"""Delegation and consult evaluation capture (architecture L9).

Each ``delegation_started`` / ``delegation_completed`` pair is already a clean
input-output record; this pairs them up and attaches a verdict, so "best model
this week" can be decided by re-running a captured set rather than by benchmark
chatter. Without it, swapping the frontier tier becomes something to avoid,
because the effect is unmeasurable.

Verdicts live outside the canonical log: the log records what happened, a
verdict is a later judgement about it, and judgements get revised.
"""

import json
from dataclasses import asdict, dataclass
from typing import Any

from nisse_dev.canonical import DelegationCompleted, DelegationStarted
from nisse_dev.state import StateStore

UNMARKED = "unmarked"


@dataclass(frozen=True)
class EvalRecord:
    event_id: int  # of the completion, which is what a verdict attaches to
    role: str
    model: str
    brief: str
    caps: dict[str, Any]
    handback: str
    artifacts: list[str]
    tokens: int
    duration_s: float
    exit_status: str
    verdict: str = UNMARKED
    note: str = ""


def collect(store: StateStore, after: int = 0) -> list[EvalRecord]:
    """Pair each start with the completion that follows it.

    A start with no completion is skipped rather than reported half-formed:
    it means the process died mid-delegation, which L8 handles, and an
    unfinished record would pollute the eval set.
    """
    verdicts = store.outcomes()
    records: list[EvalRecord] = []
    pending: tuple[int, DelegationStarted] | None = None
    for stored in store.all_events(after=after):
        if isinstance(stored.event, DelegationStarted):
            pending = (stored.id, stored.event)
        elif isinstance(stored.event, DelegationCompleted) and pending is not None:
            _, started = pending
            verdict, note = verdicts.get(stored.id, (UNMARKED, ""))
            records.append(
                EvalRecord(
                    event_id=stored.id,
                    role=started.role,
                    model=started.model,
                    brief=started.brief,
                    caps=started.caps,
                    handback=stored.event.handback,
                    artifacts=stored.event.artifacts,
                    tokens=stored.event.tokens,
                    duration_s=stored.event.duration_s,
                    exit_status=stored.event.exit_status,
                    verdict=verdict,
                    note=note,
                )
            )
            pending = None
    return records


def to_jsonl(records: list[EvalRecord]) -> str:
    """JSON lines, so a captured set can be diffed and re-run per record."""
    return "\n".join(json.dumps(asdict(r)) for r in records)


def summarise(records: list[EvalRecord]) -> dict[str, Any]:
    by_model: dict[str, dict[str, int]] = {}
    for record in records:
        bucket = by_model.setdefault(
            record.model, {"total": 0, "good": 0, "bad": 0, UNMARKED: 0}
        )
        bucket["total"] += 1
        bucket[record.verdict if record.verdict in ("good", "bad") else UNMARKED] += 1
    return by_model