The call record: every model call, for the operator's study (M6.6a)
nisse-core records each model call of every agent into a file beside its state store (<state>.calls.db), for the status page (L7, X6): - the request body exactly as sent (never headers), the raw response with its reasoning, the model asked and the one that answered, time, tokens and llama-server's own prompt reuse (timings.cache_n); - its work item, role, run and purpose: Driver.run and nisse-dev's delegation loop open runs, nested; compaction, expert consults, the classifier and the go-ahead reader label their calls; - provenance, worked out in BudgetedClient by comparing the events at each stage: each event sent with its log id and content as sent, which a compaction replaced, which fitting shortened (by how much) and dropped. Content is stored once by hash, compressed. It is diagnostic only: the agent never reads it, a failed write is logged once and recording stops while the call goes on, and the file is deleted when the agent starts, so its layout never migrates; CallReader refuses a layout it doesn't know. While recording, the system prompt ends with one sentence saying so. `call_record: false` in either agent's YAML turns it off. The benchmark records each run beside that run's own state store, and `--keep-runs DIR` keeps both for the page. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
1 parent
cba35eb7d4
commit
dd060566b2
22 files changed
+1346
-48
No files matched your search
@@ -645,3 +645,43 @@ def test_code_git_cannot_give_fails_the_run(tmp_path: Path) -> None:
|
||||
w["scheduler"].round()
|
||||
assert (item(w).position, pr_of(item(w)).attempts) == (READY, 1)
|
||||
assert w["gitea"].posted == []
|
||||
|
||||
|
||||
def test_the_review_agent_records_its_calls_unless_told_not_to(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
from nisse_core.config import ConfigError
|
||||
from nisse_core.models.catalog import ModelCatalog
|
||||
from nisse_review import build as review_build
|
||||
from nisse_review.config import ReviewConfig
|
||||
|
||||
tree = {
|
||||
"state_db": str(tmp_path / "s.db"),
|
||||
"llm": {"base_url": "http://llm", "token": "k"},
|
||||
"roles": {"reviewer": "local", "expert": "frontier"},
|
||||
"gitea": {"url": "https://g", "token": "t", "operators": ["don"]},
|
||||
}
|
||||
limits = {"max_input_tokens": 32768, "max_output_tokens": 4096}
|
||||
catalog = ModelCatalog.from_payload(
|
||||
{
|
||||
"data": [
|
||||
{
|
||||
"model_name": m,
|
||||
"litellm_params": {"model": f"x/{m}"},
|
||||
"model_info": limits,
|
||||
}
|
||||
for m in ("local", "frontier")
|
||||
]
|
||||
}
|
||||
)
|
||||
monkeypatch.setattr(review_build.ModelCatalog, "discover", lambda *a, **k: catalog)
|
||||
|
||||
on = review_build.llm_client(ReviewConfig.from_tree(tree, {}))
|
||||
assert on.call_recorder is not None
|
||||
assert on.call_recorder.path == tmp_path / "s.calls.db"
|
||||
off = review_build.llm_client(
|
||||
ReviewConfig.from_tree({**tree, "call_record": False}, {})
|
||||
)
|
||||
assert off.call_recorder is None
|
||||
with pytest.raises(ConfigError, match="call_record"):
|
||||
ReviewConfig.from_tree({**tree, "call_record": "yes"}, {})
|
||||
Reference in new issue
Block a user