nisse/nisse-dev/src/nisse_dev/evals.py

"""Delegation and consult evaluation capture (architecture L9).

Each ``delegation_started`` / ``delegation_completed`` pair is already a clean
input-output record; this pairs them up and attaches a verdict, so "best model
this week" can be decided by re-running a captured set rather than by benchmark
chatter. Without it, swapping the frontier tier becomes something to avoid,
because the effect is unmeasurable.

Verdicts live outside the canonical log: the log records what happened, a
verdict is a later judgement about it, and judgements get revised.
"""

import json
from collections.abc import Iterable
from dataclasses import asdict, dataclass
from typing import Any

from nisse_core.log.canonical import (
    ActionVerdict,
    DelegationCompleted,
    DelegationStarted,
    StoredEvent,
)
from nisse_dev.state import StateStore

UNMARKED = "unmarked"


@dataclass(frozen=True)
class EvalRecord:
    event_id: int  # of the completion, which is what a verdict attaches to
    role: str
    model: str
    brief: str
    caps: dict[str, Any]
    handback: str
    artifacts: list[str]
    tokens: int
    duration_s: float
    exit_status: str
    verdict: str = UNMARKED
    note: str = ""


def collect(store: StateStore, after: int = 0) -> list[EvalRecord]:
    """Pair each start with the completion that follows it.

    A start with no completion is skipped rather than reported half-formed:
    it means the process died mid-delegation, which L8 handles, and an
    unfinished record would pollute the eval set.
    """
    verdicts = store.outcomes()
    records: list[EvalRecord] = []
    pending: tuple[int, DelegationStarted] | None = None
    for stored in store.all_events(after=after):
        if isinstance(stored.event, DelegationStarted):
            pending = (stored.id, stored.event)
        elif isinstance(stored.event, DelegationCompleted) and pending is not None:
            _, started = pending
            verdict, note = verdicts.get(stored.id, (UNMARKED, ""))
            records.append(
                EvalRecord(
                    event_id=stored.id,
                    role=started.role,
                    model=started.model,
                    brief=started.brief,
                    caps=started.caps,
                    handback=stored.event.handback,
                    artifacts=stored.event.artifacts,
                    tokens=stored.event.tokens,
                    duration_s=stored.event.duration_s,
                    exit_status=stored.event.exit_status,
                    verdict=verdict,
                    note=note,
                )
            )
            pending = None
    return records


def to_jsonl(records: list[EvalRecord]) -> str:
    """JSON lines, so a captured set can be diffed and re-run per record."""
    return "\n".join(json.dumps(asdict(r)) for r in records)


def summarise(records: list[EvalRecord]) -> dict[str, Any]:
    by_model: dict[str, dict[str, int]] = {}
    for record in records:
        bucket = by_model.setdefault(
            record.model, {"total": 0, "good": 0, "bad": 0, UNMARKED: 0}
        )
        bucket["total"] += 1
        bucket[record.verdict if record.verdict in ("good", "bad") else UNMARKED] += 1
    return by_model


# --- the action classifier's evaluation set (L4, M3.2) -----------------------


@dataclass
class ClassifierCase:
    """One judgement, as the classifier saw it, with what then happened:
    ``overridden`` when the operator allowed what it wanted discussed."""

    input: dict[str, Any]  # exactly what the classifier was shown
    verdict: str  # safe | in_scope | needs_discussion | failed_closed
    reason: str
    model: str
    decisions: list[str]
    overridden: bool = False


def classifier_cases(events: Iterable[StoredEvent]) -> list[ClassifierCase]:
    """Every distinct classifier judgement in the log, in order. A verdict
    re-used from the cache adds its decision to the same case."""
    cases: dict[tuple[str, str], ClassifierCase] = {}
    for stored in events:
        v = stored.event
        if not isinstance(v, ActionVerdict) or v.source != "classifier":
            continue
        key = (v.input, v.rule)
        if key not in cases:
            cases[key] = ClassifierCase(
                input=json.loads(v.input) if v.input else {},
                verdict=v.rule,
                reason=v.reason,
                model=v.model,
                decisions=[],
            )
        case = cases[key]
        case.decisions.append(v.decision)
        if v.decision == "allowed":
            case.overridden = True
    return list(cases.values())