Add ask_expert and eval capture (A14/A16) — step 7c

Completes step 7.

- expert.py: one stateless call for a question needing no repo access.
  Structured output is contracted at the prompt level, since vendors
  differ on JSON enforcement, so the reply is parsed and validated
  locally with one retry — tolerating a fenced code block, because
  models wrap JSON habitually, but not missing fields
- The per-task budget is counted from the canonical log rather than
  memory, so it survives a restart and cannot hand a circling executor
  a fresh allowance. Exhaustion raises rather than answering: a caller
  out of consults should get on with the work or escalate
- Consults are recorded as delegation_started/completed with role
  consultant. A16 asks for a "consult/delegation eval set", and
  reusing the composite events gets consult evaluation for free rather
  than inventing a parallel record
- evals.py pairs starts with completions into eval records, with
  verdicts kept outside the canonical log: the log records what
  happened, a verdict is a later judgement about it, and judgements
  get revised. Schema v3 adds the outcomes table, so a v1 store now
  traverses two migrations — a better exercise of the hook than one
- ask_expert is offered to the executor as a tool only when a desk is
  configured, in addition to the observable escalation triggers and
  never instead of them (A12). Consulting is not mutating, so a
  completion claim backed only by consults is still challenged

Found live: passing an empty message list and putting the question in
the system prompt fails outright — the Messages API requires at least
one non-system message. The question now travels as a synthesised user
message that is never appended to the log, which keeps the consult
stateless while remaining a valid request.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

nisse-dev/tests/test_evals.py

@@ -0,0 +1,129 @@
+import io
+import json
+from pathlib import Path
+
+import pytest
+
+from nisse_dev.canonical import (
+    AssistantText,
+    DelegationCompleted,
+    DelegationStarted,
+)
+from nisse_dev.evals import UNMARKED, collect, summarise, to_jsonl
+from nisse_dev.oplog import OpLog
+from nisse_dev.state import StateStore
+
+
+@pytest.fixture
+def store(tmp_path: Path) -> StateStore:
+    return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
+
+
+def record_pair(
+    store: StateStore,
+    *,
+    brief: str = "add greet",
+    model: str = "claude-haiku-4-5",
+    role: str = "delegate",
+    status: str = "handback",
+) -> int:
+    store.append_event(
+        DelegationStarted(brief=brief, role=role, model=model, caps={"turns": 5})
+    )
+    store.append_event(
+        DelegationCompleted(
+            handback="what changed: greet.py",
+            artifacts=["greet.py"],
+            tokens=1200,
+            duration_s=9.5,
+            exit_status=status,
+        )
+    )
+    return store.read_events()[-1].id
+
+
+def test_pairs_a_start_with_its_completion(store: StateStore) -> None:
+    record_pair(store)
+    [record] = collect(store)
+    assert record.brief == "add greet"
+    assert record.model == "claude-haiku-4-5"
+    assert record.exit_status == "handback"
+    assert record.artifacts == ["greet.py"]
+    assert record.verdict == UNMARKED
+
+
+def test_unpaired_start_is_skipped(store: StateStore) -> None:
+    # A crash mid-delegation leaves a start with no completion; a half-formed
+    # record would pollute the set A16 exists to make trustworthy.
+    store.append_event(
+        DelegationStarted(brief="died", role="delegate", model="m", caps={})
+    )
+    assert collect(store) == []
+
+
+def test_unrelated_events_between_the_pair_are_ignored(store: StateStore) -> None:
+    store.append_event(
+        DelegationStarted(brief="b", role="delegate", model="m", caps={})
+    )
+    store.append_event(AssistantText(text="working", model="m"))
+    store.append_event(
+        DelegationCompleted(
+            handback="done",
+            artifacts=[],
+            tokens=1,
+            duration_s=1.0,
+            exit_status="handback",
+        )
+    )
+    assert len(collect(store)) == 1
+
+
+def test_verdicts_attach_to_the_completion(store: StateStore) -> None:
+    event_id = record_pair(store)
+    store.mark_outcome(event_id, "good", "clean handback, tests passed")
+    [record] = collect(store)
+    assert record.verdict == "good"
+    assert "tests passed" in record.note
+
+
+def test_a_verdict_can_be_revised(store: StateStore) -> None:
+    # Judgements get revised; the log they describe does not.
+    event_id = record_pair(store)
+    store.mark_outcome(event_id, "good")
+    store.mark_outcome(event_id, "bad", "regressed later")
+    [record] = collect(store)
+    assert record.verdict == "bad"
+    assert record.note == "regressed later"
+
+
+def test_consults_and_delegations_share_the_set(store: StateStore) -> None:
+    record_pair(store, role="delegate")
+    record_pair(store, role="consultant", brief="which approach?")
+    roles = [r.role for r in collect(store)]
+    assert roles == ["delegate", "consultant"]
+
+
+def test_collect_can_start_after_a_marker(store: StateStore) -> None:
+    record_pair(store, brief="old")
+    marker = store.read_events()[-1].id
+    record_pair(store, brief="new")
+    assert [r.brief for r in collect(store, after=marker)] == ["new"]
+
+
+def test_jsonl_round_trips(store: StateStore) -> None:
+    record_pair(store)
+    line = to_jsonl(collect(store))
+    assert json.loads(line)["model"] == "claude-haiku-4-5"
+
+
+def test_summary_groups_by_model(store: StateStore) -> None:
+    good = record_pair(store, model="haiku")
+    record_pair(store, model="haiku", status="capped")
+    record_pair(store, model="opus")
+    store.mark_outcome(good, "good")
+
+    summary = summarise(collect(store))
+    assert summary["haiku"]["total"] == 2
+    assert summary["haiku"]["good"] == 1
+    assert summary["haiku"][UNMARKED] == 1
+    assert summary["opus"]["total"] == 1