import io import json from pathlib import Path import pytest from nisse_dev.canonical import ( AssistantText, DelegationCompleted, DelegationStarted, ) from nisse_dev.evals import UNMARKED, collect, summarise, to_jsonl from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore @pytest.fixture def store(tmp_path: Path) -> StateStore: return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) def record_pair( store: StateStore, *, brief: str = "add greet", model: str = "claude-haiku-4-5", role: str = "delegate", status: str = "handback", ) -> int: store.append_event( DelegationStarted(brief=brief, role=role, model=model, caps={"turns": 5}) ) store.append_event( DelegationCompleted( handback="what changed: greet.py", artifacts=["greet.py"], tokens=1200, duration_s=9.5, exit_status=status, ) ) return store.read_events()[-1].id def test_pairs_a_start_with_its_completion(store: StateStore) -> None: record_pair(store) [record] = collect(store) assert record.brief == "add greet" assert record.model == "claude-haiku-4-5" assert record.exit_status == "handback" assert record.artifacts == ["greet.py"] assert record.verdict == UNMARKED def test_unpaired_start_is_skipped(store: StateStore) -> None: # A crash mid-delegation leaves a start with no completion; a half-formed # record would pollute the set L9 exists to make trustworthy. store.append_event( DelegationStarted(brief="died", role="delegate", model="m", caps={}) ) assert collect(store) == [] def test_unrelated_events_between_the_pair_are_ignored(store: StateStore) -> None: store.append_event( DelegationStarted(brief="b", role="delegate", model="m", caps={}) ) store.append_event(AssistantText(text="working", model="m")) store.append_event( DelegationCompleted( handback="done", artifacts=[], tokens=1, duration_s=1.0, exit_status="handback", ) ) assert len(collect(store)) == 1 def test_verdicts_attach_to_the_completion(store: StateStore) -> None: event_id = record_pair(store) store.mark_outcome(event_id, "good", "clean handback, tests passed") [record] = collect(store) assert record.verdict == "good" assert "tests passed" in record.note def test_a_verdict_can_be_revised(store: StateStore) -> None: # Judgements get revised; the log they describe does not. event_id = record_pair(store) store.mark_outcome(event_id, "good") store.mark_outcome(event_id, "bad", "regressed later") [record] = collect(store) assert record.verdict == "bad" assert record.note == "regressed later" def test_consults_and_delegations_share_the_set(store: StateStore) -> None: record_pair(store, role="delegate") record_pair(store, role="consultant", brief="which approach?") roles = [r.role for r in collect(store)] assert roles == ["delegate", "consultant"] def test_collect_can_start_after_a_marker(store: StateStore) -> None: record_pair(store, brief="old") marker = store.read_events()[-1].id record_pair(store, brief="new") assert [r.brief for r in collect(store, after=marker)] == ["new"] def test_jsonl_round_trips(store: StateStore) -> None: record_pair(store) line = to_jsonl(collect(store)) assert json.loads(line)["model"] == "claude-haiku-4-5" def test_summary_groups_by_model(store: StateStore) -> None: good = record_pair(store, model="haiku") record_pair(store, model="haiku", status="capped") record_pair(store, model="opus") store.mark_outcome(good, "good") summary = summarise(collect(store)) assert summary["haiku"]["total"] == 2 assert summary["haiku"]["good"] == 1 assert summary["haiku"][UNMARKED] == 1 assert summary["opus"]["total"] == 1