Add ask_expert and eval capture (A14/A16) — step 7c

Completes step 7.

- expert.py: one stateless call for a question needing no repo access.
  Structured output is contracted at the prompt level, since vendors
  differ on JSON enforcement, so the reply is parsed and validated
  locally with one retry — tolerating a fenced code block, because
  models wrap JSON habitually, but not missing fields
- The per-task budget is counted from the canonical log rather than
  memory, so it survives a restart and cannot hand a circling executor
  a fresh allowance. Exhaustion raises rather than answering: a caller
  out of consults should get on with the work or escalate
- Consults are recorded as delegation_started/completed with role
  consultant. A16 asks for a "consult/delegation eval set", and
  reusing the composite events gets consult evaluation for free rather
  than inventing a parallel record
- evals.py pairs starts with completions into eval records, with
  verdicts kept outside the canonical log: the log records what
  happened, a verdict is a later judgement about it, and judgements
  get revised. Schema v3 adds the outcomes table, so a v1 store now
  traverses two migrations — a better exercise of the hook than one
- ask_expert is offered to the executor as a tool only when a desk is
  configured, in addition to the observable escalation triggers and
  never instead of them (A12). Consulting is not mutating, so a
  completion claim backed only by consults is still challenged

Found live: passing an empty message list and putting the question in
the system prompt fails outright — the Messages API requires at least
one non-system message. The question now travels as a synthesised user
message that is never appended to the log, which keeps the consult
stateless while remaining a valid request.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

nisse-dev/tests/test_canonical.py

@@ -16,7 +16,7 @@ from nisse_dev.canonical import (
     from_payload,
     mint_call_id,
 )
-from nisse_dev.state import StateStore
+from nisse_dev.state import CURRENT_SCHEMA_VERSION, StateStore
 
 ALL_EVENTS = [
     UserMessage(text="use demo", source="matrix:$abc"),
@@ -116,9 +116,12 @@ def test_default_migration_upgrades_a_real_v1_store(tmp_path: Path) -> None:
     conn.close()
 
     with StateStore(db) as store:
-        assert store.schema_version == 2
+        assert store.schema_version == CURRENT_SCHEMA_VERSION
         state = store.get_state()
         assert state is not None
         assert state.state == "IDLE"  # v1 data intact
         store.append_event(UserMessage(text="hello", source="m:1"))
         assert len(store.read_events()) == 1
+        # Two migrations were traversed, so both later tables must exist.
+        store.mark_outcome(1, "good", "arrived via v1 -> v3")
+        assert store.outcomes()[1] == ("good", "arrived via v1 -> v3")

nisse-dev/tests/test_evals.py

@@ -0,0 +1,129 @@
+import io
+import json
+from pathlib import Path
+
+import pytest
+
+from nisse_dev.canonical import (
+    AssistantText,
+    DelegationCompleted,
+    DelegationStarted,
+)
+from nisse_dev.evals import UNMARKED, collect, summarise, to_jsonl
+from nisse_dev.oplog import OpLog
+from nisse_dev.state import StateStore
+
+
+@pytest.fixture
+def store(tmp_path: Path) -> StateStore:
+    return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
+
+
+def record_pair(
+    store: StateStore,
+    *,
+    brief: str = "add greet",
+    model: str = "claude-haiku-4-5",
+    role: str = "delegate",
+    status: str = "handback",
+) -> int:
+    store.append_event(
+        DelegationStarted(brief=brief, role=role, model=model, caps={"turns": 5})
+    )
+    store.append_event(
+        DelegationCompleted(
+            handback="what changed: greet.py",
+            artifacts=["greet.py"],
+            tokens=1200,
+            duration_s=9.5,
+            exit_status=status,
+        )
+    )
+    return store.read_events()[-1].id
+
+
+def test_pairs_a_start_with_its_completion(store: StateStore) -> None:
+    record_pair(store)
+    [record] = collect(store)
+    assert record.brief == "add greet"
+    assert record.model == "claude-haiku-4-5"
+    assert record.exit_status == "handback"
+    assert record.artifacts == ["greet.py"]
+    assert record.verdict == UNMARKED
+
+
+def test_unpaired_start_is_skipped(store: StateStore) -> None:
+    # A crash mid-delegation leaves a start with no completion; a half-formed
+    # record would pollute the set A16 exists to make trustworthy.
+    store.append_event(
+        DelegationStarted(brief="died", role="delegate", model="m", caps={})
+    )
+    assert collect(store) == []
+
+
+def test_unrelated_events_between_the_pair_are_ignored(store: StateStore) -> None:
+    store.append_event(
+        DelegationStarted(brief="b", role="delegate", model="m", caps={})
+    )
+    store.append_event(AssistantText(text="working", model="m"))
+    store.append_event(
+        DelegationCompleted(
+            handback="done",
+            artifacts=[],
+            tokens=1,
+            duration_s=1.0,
+            exit_status="handback",
+        )
+    )
+    assert len(collect(store)) == 1
+
+
+def test_verdicts_attach_to_the_completion(store: StateStore) -> None:
+    event_id = record_pair(store)
+    store.mark_outcome(event_id, "good", "clean handback, tests passed")
+    [record] = collect(store)
+    assert record.verdict == "good"
+    assert "tests passed" in record.note
+
+
+def test_a_verdict_can_be_revised(store: StateStore) -> None:
+    # Judgements get revised; the log they describe does not.
+    event_id = record_pair(store)
+    store.mark_outcome(event_id, "good")
+    store.mark_outcome(event_id, "bad", "regressed later")
+    [record] = collect(store)
+    assert record.verdict == "bad"
+    assert record.note == "regressed later"
+
+
+def test_consults_and_delegations_share_the_set(store: StateStore) -> None:
+    record_pair(store, role="delegate")
+    record_pair(store, role="consultant", brief="which approach?")
+    roles = [r.role for r in collect(store)]
+    assert roles == ["delegate", "consultant"]
+
+
+def test_collect_can_start_after_a_marker(store: StateStore) -> None:
+    record_pair(store, brief="old")
+    marker = store.read_events()[-1].id
+    record_pair(store, brief="new")
+    assert [r.brief for r in collect(store, after=marker)] == ["new"]
+
+
+def test_jsonl_round_trips(store: StateStore) -> None:
+    record_pair(store)
+    line = to_jsonl(collect(store))
+    assert json.loads(line)["model"] == "claude-haiku-4-5"
+
+
+def test_summary_groups_by_model(store: StateStore) -> None:
+    good = record_pair(store, model="haiku")
+    record_pair(store, model="haiku", status="capped")
+    record_pair(store, model="opus")
+    store.mark_outcome(good, "good")
+
+    summary = summarise(collect(store))
+    assert summary["haiku"]["total"] == 2
+    assert summary["haiku"]["good"] == 1
+    assert summary["haiku"][UNMARKED] == 1
+    assert summary["opus"]["total"] == 1

nisse-dev/tests/test_executor.py

@@ -223,3 +223,95 @@ def test_multiple_calls_in_one_turn_all_run(store: StateStore, ws: Workspace) ->
         e.event for e in store.read_events() if type(e.event).__name__ == "ToolResult"
     ]
     assert {r.call_id for r in results} == {"a", "b"}
+
+
+class StubDesk:
+    """Stands in for ExpertDesk without a model behind it."""
+
+    def __init__(self, answer: str = "use sqlite", error: Exception | None = None):
+        self.asked: list[str] = []
+        self._answer = answer
+        self._error = error
+
+    def ask(self, question: str, **kw: Any) -> Any:
+        self.asked.append(question)
+        if self._error is not None:
+            raise self._error
+
+        class _A:
+            def render(self_inner) -> str:
+                return self._answer
+
+        return _A()
+
+
+def test_ask_expert_is_offered_only_when_a_desk_exists(
+    store: StateStore, ws: Workspace
+) -> None:
+    client = ScriptedClient([])
+    build(client, store, ws).run(milestone=1)
+    assert all(t["function"]["name"] != "ask_expert" for t in client.calls[0]["tools"])
+
+    client2 = ScriptedClient([])
+    build(client2, store, ws, expert=StubDesk()).run(milestone=1)
+    names = [t["function"]["name"] for t in client2.calls[0]["tools"]]
+    assert names[-1] == "ask_expert"
+    # A12: in addition to the observable triggers, not instead of them.
+    assert "run_command" in names
+
+
+def test_ask_expert_answer_comes_back_as_a_tool_result(
+    store: StateStore, ws: Workspace
+) -> None:
+    desk = StubDesk(answer="prefer sqlite here")
+    client = ScriptedClient(
+        [
+            [call("ask_expert", question="sqlite or postgres?")],
+            [call("write_file", path="a.py", content="x")],
+            [AssistantText(text="done", model="m")],
+        ]
+    )
+    build(client, store, ws, expert=desk).run(milestone=1)
+
+    assert desk.asked == ["sqlite or postgres?"]
+    results = [
+        e.event for e in store.read_events() if type(e.event).__name__ == "ToolResult"
+    ]
+    assert "prefer sqlite here" in results[0].result
+
+
+def test_consulting_is_not_doing_the_work(store: StateStore, ws: Workspace) -> None:
+    # Asking is not changing anything, so a completion claim backed only by
+    # consults is still challenged.
+    client = ScriptedClient(
+        [
+            [call("ask_expert", question="what should I do?")],
+            [AssistantText(text="I think it is done", model="m")],
+            [AssistantText(text="really done", model="m")],
+        ]
+    )
+    outcome = build(client, store, ws, expert=StubDesk()).run(milestone=1)
+    assert outcome.status == "escalate"
+
+
+def test_exhausted_budget_is_reported_to_the_model_not_raised(
+    store: StateStore, ws: Workspace
+) -> None:
+    from nisse_dev.expert import BudgetExhausted
+
+    desk = StubDesk(error=BudgetExhausted("consult budget of 3 is spent"))
+    client = ScriptedClient(
+        [
+            [call("ask_expert", question="again?")],
+            [call("write_file", path="a.py", content="x")],
+            [AssistantText(text="done", model="m")],
+        ]
+    )
+    outcome = build(client, store, ws, expert=desk).run(milestone=1)
+
+    assert outcome.status == "completed"  # it got on with the work
+    results = [
+        e.event for e in store.read_events() if type(e.event).__name__ == "ToolResult"
+    ]
+    assert not results[0].success
+    assert "budget" in results[0].result

nisse-dev/tests/test_expert.py

@@ -0,0 +1,186 @@
+import io
+from pathlib import Path
+from typing import Any
+
+import pytest
+
+from nisse_dev.canonical import (
+    AssistantText,
+    CanonicalEvent,
+    DelegationCompleted,
+    DelegationStarted,
+    StoredEvent,
+)
+from nisse_dev.expert import (
+    DEFAULT_BUDGET,
+    BudgetExhausted,
+    ExpertAnswer,
+    ExpertDesk,
+    parse_answer,
+)
+from nisse_dev.llm import LLMError, RoleMap
+from nisse_dev.oplog import OpLog
+from nisse_dev.state import StateStore
+
+ROLES = RoleMap(
+    executor="llama3.2",
+    planner="haiku",
+    delegate="haiku",
+    consultant="claude-haiku-4-5",
+)
+
+GOOD = '{"answer": "Use SQLite; it is enough here.", "confidence": "high", "caveats": "Assumes one writer."}'
+
+
+class ScriptedClient:
+    def __init__(self, replies: list[str]) -> None:
+        self._replies = replies
+        self.calls: list[dict[str, Any]] = []
+
+    def complete(
+        self,
+        events: list[StoredEvent],
+        tools: list[dict[str, Any]] | None,
+        model: str,
+        system: str | None = None,
+    ) -> list[CanonicalEvent]:
+        self.calls.append(
+            {"events": events, "tools": tools, "model": model, "system": system}
+        )
+        if not self._replies:
+            return [AssistantText(text="nothing more", model=model)]
+        return [AssistantText(text=self._replies.pop(0), model=model)]
+
+
+@pytest.fixture
+def store(tmp_path: Path) -> StateStore:
+    return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
+
+
+def desk(client: Any, store: StateStore, **kw: Any) -> ExpertDesk:
+    return ExpertDesk(client, store, ROLES, **kw)
+
+
+# --- parsing (the contract is prompt-level, so parsing is the guard) ------
+
+
+def test_parses_a_clean_object() -> None:
+    parsed = parse_answer(GOOD)
+    assert parsed == ExpertAnswer(
+        answer="Use SQLite; it is enough here.",
+        confidence="high",
+        caveats="Assumes one writer.",
+    )
+
+
+def test_parses_a_fenced_block_because_models_wrap_json() -> None:
+    assert parse_answer(f"Here you go:\n```json\n{GOOD}\n```") is not None
+
+
+def test_missing_or_bad_fields_are_a_failed_turn() -> None:
+    assert parse_answer('{"answer": "", "confidence": "high"}') is None
+    assert parse_answer('{"answer": "x", "confidence": "certain"}') is None
+    assert parse_answer('{"confidence": "high"}') is None
+    assert parse_answer("no json at all") is None
+    assert parse_answer('["not", "an", "object"]') is None
+
+
+def test_absent_caveats_is_tolerated() -> None:
+    parsed = parse_answer('{"answer": "yes", "confidence": "low"}')
+    assert parsed is not None
+    assert parsed.caveats == ""
+
+
+# --- the desk ------------------------------------------------------------
+
+
+def test_ask_returns_a_validated_answer(store: StateStore) -> None:
+    client = ScriptedClient([GOOD])
+    answer = desk(client, store).ask("SQLite or Postgres?")
+    assert answer.confidence == "high"
+    assert "SQLite" in answer.render()
+
+
+def test_consult_is_stateless_and_uses_the_consultant_role(
+    store: StateStore,
+) -> None:
+    store.append_event(AssistantText(text="secret repo context", model="m"))
+    client = ScriptedClient([GOOD])
+    desk(client, store).ask("Is this API deprecated?")
+
+    call = client.calls[0]
+    # A14: no repo access, no session state — exactly the question, and none
+    # of the log the caller was working from.
+    assert [e.event.text for e in call["events"]] == ["Is this API deprecated?"]
+    assert all("secret repo context" not in e.event.text for e in call["events"])
+    assert call["tools"] is None
+    assert call["model"] == "claude-haiku-4-5"
+
+
+def test_unparseable_reply_is_retried_once_then_raises(store: StateStore) -> None:
+    client = ScriptedClient(["waffle", "more waffle"])
+    with pytest.raises(LLMError, match="no usable answer"):
+        desk(client, store).ask("q")
+    assert len(client.calls) == 2
+    assert "not a single valid JSON object" in client.calls[1]["system"]
+
+
+def test_retry_succeeds_after_a_bad_first_reply(store: StateStore) -> None:
+    client = ScriptedClient(["waffle", GOOD])
+    assert desk(client, store).ask("q").confidence == "high"
+
+
+def test_consults_are_recorded_as_composite_events(store: StateStore) -> None:
+    # A16 wants a consult/delegation eval set; reusing the pair gets consult
+    # evaluation for free.
+    desk(ScriptedClient([GOOD]), store).ask("q")
+    events = [e.event for e in store.read_events()]
+    started = next(e for e in events if isinstance(e, DelegationStarted))
+    completed = next(e for e in events if isinstance(e, DelegationCompleted))
+    assert started.role == "consultant"
+    assert started.brief == "q"
+    assert completed.exit_status == "answered"
+
+
+def test_a_failed_consult_is_recorded_too(store: StateStore) -> None:
+    with pytest.raises(LLMError):
+        desk(ScriptedClient(["waffle", "waffle"]), store).ask("q")
+    completed = [
+        e.event for e in store.read_events() if isinstance(e.event, DelegationCompleted)
+    ]
+    assert completed[0].exit_status == "failed"
+
+
+def test_budget_is_enforced_across_calls(store: StateStore) -> None:
+    client = ScriptedClient([GOOD] * 10)
+    d = desk(client, store, budget=2)
+    d.ask("one")
+    d.ask("two")
+    with pytest.raises(BudgetExhausted, match="budget of 2"):
+        d.ask("three")
+    assert len(client.calls) == 2  # the third never reached the model
+
+
+def test_budget_survives_a_restart(tmp_path: Path) -> None:
+    # Counted from the canonical log, not from memory, so a crash cannot
+    # hand a circling executor a fresh allowance.
+    db = tmp_path / "state.db"
+    with StateStore(db, oplog=OpLog("d", io.StringIO())) as store:
+        desk(ScriptedClient([GOOD]), store, budget=1).ask("one")
+    with (
+        StateStore(db, oplog=OpLog("d", io.StringIO())) as store,
+        pytest.raises(BudgetExhausted),
+    ):
+        desk(ScriptedClient([GOOD]), store, budget=1).ask("two")
+
+
+def test_budget_counts_only_after_the_task_marker(store: StateStore) -> None:
+    d = desk(ScriptedClient([GOOD] * 5), store, budget=1)
+    d.ask("earlier task")
+    marker = store.read_events()[-1].id
+    # A new task starts its allowance afresh from its own marker.
+    assert d.ask("new task", since_event_id=marker).confidence == "high"
+
+
+def test_default_budget_is_small() -> None:
+    assert DEFAULT_BUDGET <= 5  # A14: do not let this become a second subsystem