The call record: every model call, for the operator's study (M6.6a)

nisse-core records each model call of every agent into a file beside its
state store (<state>.calls.db), for the status page (L7, X6):

- the request body exactly as sent (never headers), the raw response with
  its reasoning, the model asked and the one that answered, time, tokens
  and llama-server's own prompt reuse (timings.cache_n);
- its work item, role, run and purpose: Driver.run and nisse-dev's
  delegation loop open runs, nested; compaction, expert consults, the
  classifier and the go-ahead reader label their calls;
- provenance, worked out in BudgetedClient by comparing the events at
  each stage: each event sent with its log id and content as sent, which
  a compaction replaced, which fitting shortened (by how much) and
  dropped.

Content is stored once by hash, compressed. It is diagnostic only: the
agent never reads it, a failed write is logged once and recording stops
while the call goes on, and the file is deleted when the agent starts,
so its layout never migrates; CallReader refuses a layout it doesn't
know. While recording, the system prompt ends with one sentence saying
so. `call_record: false` in either agent's YAML turns it off.

The benchmark records each run beside that run's own state store, and
`--keep-runs DIR` keeps both for the page.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk

nisse-dev/src/nisse_dev/bench/runner.py

@@ -1,11 +1,13 @@
 """Running one task once with one driver model, and recording it."""
 
+import contextlib
 import io
 import shlex
 import subprocess
 import tempfile
 import time
 import uuid
+from collections.abc import Iterator
 from dataclasses import asdict, dataclass, field
 from datetime import UTC, datetime
 from pathlib import Path
@@ -21,6 +23,7 @@ from nisse_core.log.canonical import (
     UserMessage,
 )
 from nisse_core.loop.driver import Driver, RunOutcome
+from nisse_core.models import calls
 from nisse_core.models.catalog import ModelFacts
 from nisse_core.models.compact import Compactor
 from nisse_core.oplog import OpLog
@@ -184,6 +187,37 @@ def _count(events: list[StoredEvent]) -> tuple[int, int]:
     return failed, invented
 
 
+@contextlib.contextmanager
+def _run_dir(keep: Path | None, workdir: Path | None, name: str) -> Iterator[Path]:
+    """Where a run's state store and call record go: kept under ``keep``
+    for the status page (X6), or a temporary directory."""
+    if keep is not None:
+        root = keep / f"{name}-{uuid.uuid4().hex[:8]}"
+        root.mkdir(parents=True)
+        yield root
+        return
+    with tempfile.TemporaryDirectory(dir=workdir) as tmp:
+        yield Path(tmp)
+
+
+@contextlib.contextmanager
+def _recording(client: Any, root: Path) -> Iterator[None]:
+    """The run's calls recorded beside its own state store (L7), as the
+    agent records its own, so the run measures what the agent would do."""
+    recorder = calls.CallRecorder(calls.path_for(root / "state.db"))
+    had = hasattr(client, "call_recorder")
+    previous = getattr(client, "call_recorder", None)
+    client.call_recorder = recorder
+    try:
+        yield
+    finally:
+        if had:
+            client.call_recorder = previous
+        else:
+            del client.call_recorder
+        recorder.close()
+
+
 def run_task(
     task: Task,
     facts: ModelFacts,
@@ -194,6 +228,7 @@ def run_task(
     note: str = "",
     commit: str = "",
     workdir: Path | None = None,
+    keep: Path | None = None,
 ) -> RunRecord:
     """Run one task once: seed a workspace, hand the driver an agreed item
     whose restatement is the task's plan, then judge the result with the
@@ -202,8 +237,7 @@ def run_task(
     It is the loop's own driver, system prompt and tools, so what is measured
     is what the agent would do, without the harness around it: the milestone
     is started but no branch is cut, and finishing it ends the run."""
-    with tempfile.TemporaryDirectory(dir=workdir) as tmp:
-        root = Path(tmp)
+    with _run_dir(keep, workdir, task.name) as root, _recording(client, root):
         # The workspace is on the build VM, like the agent's own (L13).
         workspace = Workspace(
             host,

nisse-dev/src/nisse_dev/config.py

@@ -63,6 +63,8 @@ OPTIONAL_KEYS = (
     "roles.classifier",
     "llm.max_output_tokens",
     "llm.request_timeout_s",
+    # Record every model call for the operator's study (L7); on unless false.
+    "call_record",
 )
 # Settings the old loop had (until M2.8), and what to do instead.
 RETIRED_KEYS = {
@@ -444,6 +446,7 @@ class AgentConfig:
     profiles: Mapping[str, Profile] = field(default_factory=dict)
     max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS
     request_timeout_s: float = DEFAULT_REQUEST_TIMEOUT_S
+    call_record: bool = True  # every model call recorded for the operator (L7)
     tuning: Tuning = field(default_factory=Tuning)
     vault: VaultConfig | None = None
     config_file: Path | None = None
@@ -521,6 +524,9 @@ class AgentConfig:
             errors.append("unset or empty — " + ", ".join(missing))
 
         max_output, request_timeout = llm_numbers(values, errors)
+        record = values.get("call_record", True)
+        if not isinstance(record, bool):
+            errors.append("call_record: true or false")
         if errors:
             raise ConfigError("cannot start: " + "; ".join(errors))
 
@@ -548,6 +554,7 @@ class AgentConfig:
             profiles=profiles,
             max_output_tokens=max_output,
             request_timeout_s=request_timeout,
+            call_record=record,
             tuning=tuning,
             vault=vault,
             config_file=config_file,

nisse-dev/src/nisse_dev/delegation.py

@@ -38,6 +38,7 @@ from nisse_core.log.canonical import (
     ToolCall,
     ToolResult,
 )
+from nisse_core.models import calls
 from nisse_core.models.llm import LLMClient, LLMError, UpstreamUnavailable
 from nisse_core.state.cancel import Cancel, skip_calls
 from nisse_core.state.reconcile import Reconciler
@@ -333,6 +334,21 @@ class Delegation:
 
     def _loop(
         self, transcript: Transcript, brief: str, branch: str, started_at: float
+    ) -> DelegationResult:
+        """The delegation's turns, recorded as a run of its own, inside the
+        driver's (L7)."""
+        with calls.run(
+            calls.recorder_of(self._client),
+            work_item=transcript.work_item,
+            role=transcript.actor,
+            kind="delegation",
+        ) as recorded:
+            result = self._turns(transcript, brief, branch, started_at)
+            recorded.end(result.status, result.note)
+            return result
+
+    def _turns(
+        self, transcript: Transcript, brief: str, branch: str, started_at: float
     ) -> DelegationResult:
         system = (
             f"{SYSTEM_PROMPT.format(reader=self._reader)}\n\n"

nisse-dev/src/nisse_dev/loop/build.py

@@ -14,6 +14,7 @@ from nisse_core.gitea import GiteaClient
 from nisse_core.knowledge.memory import Memory
 from nisse_core.knowledge.vault import Vault
 from nisse_core.loop.driver import Driver
+from nisse_core.models import calls
 from nisse_core.models.catalog import ModelCatalog, select_client
 from nisse_core.models.compact import Compactor
 from nisse_core.models.context import BudgetedClient
@@ -126,6 +127,9 @@ def make_llm_clients(
         assert facts.max_input_tokens is not None  # require_limits checked
         return facts.max_input_tokens, facts.max_output_tokens
 
+    # Both routes record into one file (L7).
+    recorder = calls.recorder_for(config.state_db, config.call_record, oplog)
+
     def budgeted(client: object) -> BudgetedClient:
         # Every request, on either route, is fitted to its model's window.
         return BudgetedClient(
@@ -138,6 +142,7 @@ def make_llm_clients(
                 catalog.facts(m).supports_tool_choice is True
             ),
             oplog=oplog,  # a cut that fitting has to make is logged
+            call_recorder=recorder,
         )
 
     normalized = budgeted(

nisse-dev/src/nisse_dev/loop/classifier.py

@@ -18,6 +18,7 @@ from typing import Any
 
 from nisse_core.authority.actions import CONTAINED, DANGEROUS, Action, Verdict
 from nisse_core.log.canonical import StoredEvent, ToolCall, UserMessage
+from nisse_core.models.calls import purpose
 from nisse_core.models.llm import LLMClient, LLMError
 
 SAFE = "safe"
@@ -120,19 +121,20 @@ class ActionClassifier:
         shown = f"{model} ({resolved})" if resolved and resolved != model else model
         prompt = self.input_for(action, scope, workspace)
         try:
-            produced = self._client.complete(
-                [
-                    StoredEvent(
-                        id=0,
-                        ts="",
-                        event=UserMessage(text=prompt, source="action classifier"),
-                        actor="orchestrator",
-                    )
-                ],
-                tools=[VERDICT],
-                model=model,
-                system=SYSTEM,
-            )
+            with purpose("classification", role="classifier"):
+                produced = self._client.complete(
+                    [
+                        StoredEvent(
+                            id=0,
+                            ts="",
+                            event=UserMessage(text=prompt, source="action classifier"),
+                            actor="orchestrator",
+                        )
+                    ],
+                    tools=[VERDICT],
+                    model=model,
+                    system=SYSTEM,
+                )
         except LLMError as exc:
             return _closed(f"the classifier's call failed: {exc}"), shown
         except Exception as exc:  # noqa: BLE001 -- fails closed, whatever it was

nisse-dev/src/nisse_dev/loop/reader.py

@@ -11,6 +11,7 @@ The driver's model reads for now; the classifier role takes over in M2.6.
 from typing import Any
 
 from nisse_core.log.canonical import StoredEvent, ToolCall, UserMessage
+from nisse_core.models.calls import purpose
 from nisse_core.models.llm import LLMClient, LLMError
 
 
@@ -65,19 +66,20 @@ class GoAheadReader:
             return False
         prompt = f"The proposal:\n\n{proposal}\n\nThe operator's reply:\n\n{reply}"
         try:
-            produced = self._client.complete(
-                [
-                    StoredEvent(
-                        id=0,
-                        ts="",
-                        event=UserMessage(text=prompt, source="go-ahead reader"),
-                        actor="orchestrator",
-                    )
-                ],
-                tools=[VERDICT],
-                model=model or self._model,
-                system=SYSTEM,
-            )
+            with purpose("go-ahead reading", role="classifier"):
+                produced = self._client.complete(
+                    [
+                        StoredEvent(
+                            id=0,
+                            ts="",
+                            event=UserMessage(text=prompt, source="go-ahead reader"),
+                            actor="orchestrator",
+                        )
+                    ],
+                    tools=[VERDICT],
+                    model=model or self._model,
+                    system=SYSTEM,
+                )
         except LLMError:
             return False  # unread: the driver reads it as an answer instead
         return any(_yes(event) for event in produced)

nisse-dev/tests/test_bench.py

@@ -429,3 +429,26 @@ def test_runs_from_before_the_driver_read_as_its_defaults() -> None:
     old["tuning"]["executor"] = old["tuning"].pop("driver")
     [row] = summarise([old], current)
     assert row.tuning == ""
+
+
+def test_a_kept_run_leaves_its_state_and_call_record(tmp_path: Path) -> None:
+    # As the agent records itself (L7), so the status page can open a
+    # benchmark run (X6); each run gets its own record, and the client is
+    # left as it was.
+    from nisse_core.models.calls import CallReader
+
+    [task] = load_tasks(["write-function"])
+    client = Scripted(list(SOLVE_SLUGIFY))
+    record = run_task(task, FACTS, client, Tuning(), host=HOST, keep=tmp_path / "runs")
+
+    [kept] = list((tmp_path / "runs").iterdir())
+    assert kept.name.startswith("write-function-")
+    assert (kept / "state.db").exists()
+    [run] = CallReader(kept / "state.calls.db").runs()
+    assert (run["work_item"], run["role"], run["outcome"]) == (
+        "bench",
+        "driver",
+        "wait",
+    )
+    assert record.success
+    assert not hasattr(client, "call_recorder")

nisse-dev/tests/test_config.py

@@ -573,3 +573,28 @@ def test_the_vault_is_read() -> None:
 def test_a_bad_vault_is_refused(vault: dict[str, Any], said: str) -> None:
     with pytest.raises(ConfigError, match=re.escape(said)):
         build(vault=vault)
+
+
+def test_the_call_record_is_on_unless_turned_off() -> None:
+    assert build().call_record is True
+    assert build(call_record=False).call_record is False
+    with pytest.raises(ConfigError, match="call_record"):
+        build(call_record="no")
+
+
+def test_both_routes_record_into_one_file_beside_the_state_store(
+    tmp_path: Path,
+) -> None:
+    from nisse_core.models.catalog import ModelCatalog
+    from nisse_dev.loop.build import make_llm_clients
+
+    normalized, passthrough = make_llm_clients(
+        build(state_db=str(tmp_path / "state.db")), ModelCatalog({})
+    )
+    assert normalized.call_recorder is passthrough.call_recorder
+    assert normalized.call_recorder.path == tmp_path / "state.calls.db"
+    off, _ = make_llm_clients(
+        build(state_db=str(tmp_path / "other.db"), call_record=False), ModelCatalog({})
+    )
+    assert off.call_recorder is None
+    assert not (tmp_path / "other.calls.db").exists()

nisse-dev/tests/test_delegation.py

@@ -702,3 +702,25 @@ def test_a_bad_request_is_not_given_the_benefit_of_the_doubt(
     result = build(BadRequestClient([]), store, ws, repo).run(brief="b", branch="d")
     assert result.status == "failed"
     assert not result.interrupted
+
+
+def test_a_delegation_is_recorded_as_a_run_of_its_own(
+    store: StateStore, ws: Workspace, repo: GitRepo
+) -> None:
+    # Its calls are the helper's turns (L7), inside the run that delegated.
+    from nisse_core.models import calls
+
+    seen: list[calls.Scope] = []
+
+    class Watching(ScriptedClient):
+        def complete(self, *args: Any, **kwargs: Any) -> list[CanonicalEvent]:
+            seen.append(calls.current())
+            return super().complete(*args, **kwargs)
+
+    client = Watching([[hand_back(succeeded=False, changed="nothing")]])
+    with calls.run(None, work_item=ITEM, role="driver"):
+        outer = calls.current().run
+        build(client, store, ws, repo).run(brief="look into it", branch="d1")
+    [scope] = seen
+    assert (scope.role, scope.work_item, scope.parent) == ("helper", ITEM, outer)
+    assert scope.run not in (None, outer)

nisse-dev/tests/test_loop_classifier.py

@@ -346,3 +346,27 @@ def test_the_export_script_reads_a_store(tmp_path: Path) -> None:
     lines = [json.loads(line) for line in run.stdout.splitlines()]
     assert [line["verdict"] for line in lines] == ["needs_discussion", "safe"]
     assert lines[0]["input"]["action"]["arguments"] == {"command": "x"}
+
+
+def test_the_classifiers_calls_are_recorded_as_its_own() -> None:
+    # In the call record (L7), a verdict is the classifier's call, not a
+    # turn of the run that asked for it.
+    from nisse_core.models import calls
+
+    seen: list[calls.Scope] = []
+
+    class Watching(Judge):
+        def complete(self, events, tools, model, system=None) -> list[Any]:
+            seen.append(calls.current())
+            return super().complete(events, tools, model, system)
+
+    classifier = ActionClassifier(Watching(verdict("contained")), "local")  # type: ignore[arg-type]
+    with calls.run(None, work_item="w", role="driver"):
+        classifier.judge(Action("run_command", {"command": "make"}), SCOPE, "/w")
+    [scope] = seen
+    assert (scope.role, scope.purpose, scope.work_item) == (
+        "classifier",
+        "classification",
+        "w",
+    )
+    assert scope.run is not None

nisse-dev/tests/test_loop_labels.py

@@ -316,3 +316,19 @@ def test_a_restatement_that_asks_is_sent_back(tmp_path: Path) -> None:
         if hasattr(e.event, "result")
     ]
     assert any("a restatement asks no questions" in r for r in results)
+
+
+def test_the_go_ahead_readers_calls_are_recorded_as_the_classifiers() -> None:
+    from nisse_core.models import calls
+
+    seen: list[calls.Scope] = []
+
+    class Watching(_Model):
+        def complete(self, *args: Any, **kwargs: Any) -> Any:
+            seen.append(calls.current())
+            return super().complete(*args, **kwargs)
+
+    yes = [ToolCall(call_id="v", tool="verdict", arguments={"answer": "yes"})]
+    GoAheadReader(Watching(yes), "m").agrees("add count_words", "ok")  # type: ignore[arg-type]
+    [scope] = seen
+    assert (scope.role, scope.purpose) == ("classifier", "go-ahead reading")