dongarandClaude Opus 5.5
e351d0a74e
A pure move; behaviour unchanged. - models (L9, L7): llm, catalog, probe, render, render_anthropic, context - log (L7, L6): canonical, views - state (L8): store (was state), reconcile, worker, cancel, dump - knowledge (L10, L6.4): vault, memory - conversation (L5): transport, matrix - execution (L13): ssh (was execution), testing - authority (P3, P4, L4): actions - gitea, oplog, envfile stay at the top until each gains a second module. Every import, monkeypatch target, script and doc reference rewritten. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
130 lines
4.0 KiB
Python
130 lines
4.0 KiB
Python
import io
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from nisse_core.log.canonical import (
|
|
AssistantText,
|
|
DelegationCompleted,
|
|
DelegationStarted,
|
|
)
|
|
from nisse_core.oplog import OpLog
|
|
from nisse_dev.evals import UNMARKED, collect, summarise, to_jsonl
|
|
from nisse_dev.state import StateStore
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> StateStore:
|
|
return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
|
|
|
|
|
def record_pair(
|
|
store: StateStore,
|
|
*,
|
|
brief: str = "add greet",
|
|
model: str = "claude-haiku-4-5",
|
|
role: str = "helper",
|
|
status: str = "handback",
|
|
) -> int:
|
|
store.transcript("t", "helper").append(
|
|
DelegationStarted(brief=brief, role=role, model=model, caps={"turns": 5})
|
|
)
|
|
store.transcript("t", "helper").append(
|
|
DelegationCompleted(
|
|
handback="what changed: greet.py",
|
|
artifacts=["greet.py"],
|
|
tokens=1200,
|
|
duration_s=9.5,
|
|
exit_status=status,
|
|
)
|
|
)
|
|
return store.all_events()[-1].id
|
|
|
|
|
|
def test_pairs_a_start_with_its_completion(store: StateStore) -> None:
|
|
record_pair(store)
|
|
[record] = collect(store)
|
|
assert record.brief == "add greet"
|
|
assert record.model == "claude-haiku-4-5"
|
|
assert record.exit_status == "handback"
|
|
assert record.artifacts == ["greet.py"]
|
|
assert record.verdict == UNMARKED
|
|
|
|
|
|
def test_unpaired_start_is_skipped(store: StateStore) -> None:
|
|
# A crash mid-delegation leaves a start with no completion; a half-formed
|
|
# record would pollute the set L9 exists to make trustworthy.
|
|
store.transcript("t", "helper").append(
|
|
DelegationStarted(brief="died", role="helper", model="m", caps={})
|
|
)
|
|
assert collect(store) == []
|
|
|
|
|
|
def test_unrelated_events_between_the_pair_are_ignored(store: StateStore) -> None:
|
|
store.transcript("t", "helper").append(
|
|
DelegationStarted(brief="b", role="helper", model="m", caps={})
|
|
)
|
|
store.transcript("t", "helper").append(AssistantText(text="working", model="m"))
|
|
store.transcript("t", "helper").append(
|
|
DelegationCompleted(
|
|
handback="done",
|
|
artifacts=[],
|
|
tokens=1,
|
|
duration_s=1.0,
|
|
exit_status="handback",
|
|
)
|
|
)
|
|
assert len(collect(store)) == 1
|
|
|
|
|
|
def test_verdicts_attach_to_the_completion(store: StateStore) -> None:
|
|
event_id = record_pair(store)
|
|
store.mark_outcome(event_id, "good", "clean handback, tests passed")
|
|
[record] = collect(store)
|
|
assert record.verdict == "good"
|
|
assert "tests passed" in record.note
|
|
|
|
|
|
def test_a_verdict_can_be_revised(store: StateStore) -> None:
|
|
# Judgements get revised; the log they describe does not.
|
|
event_id = record_pair(store)
|
|
store.mark_outcome(event_id, "good")
|
|
store.mark_outcome(event_id, "bad", "regressed later")
|
|
[record] = collect(store)
|
|
assert record.verdict == "bad"
|
|
assert record.note == "regressed later"
|
|
|
|
|
|
def test_consults_and_delegations_share_the_set(store: StateStore) -> None:
|
|
record_pair(store, role="helper")
|
|
record_pair(store, role="expert", brief="which approach?")
|
|
roles = [r.role for r in collect(store)]
|
|
assert roles == ["helper", "expert"]
|
|
|
|
|
|
def test_collect_can_start_after_a_marker(store: StateStore) -> None:
|
|
record_pair(store, brief="old")
|
|
marker = store.all_events()[-1].id
|
|
record_pair(store, brief="new")
|
|
assert [r.brief for r in collect(store, after=marker)] == ["new"]
|
|
|
|
|
|
def test_jsonl_round_trips(store: StateStore) -> None:
|
|
record_pair(store)
|
|
line = to_jsonl(collect(store))
|
|
assert json.loads(line)["model"] == "claude-haiku-4-5"
|
|
|
|
|
|
def test_summary_groups_by_model(store: StateStore) -> None:
|
|
good = record_pair(store, model="haiku")
|
|
record_pair(store, model="haiku", status="capped")
|
|
record_pair(store, model="opus")
|
|
store.mark_outcome(good, "good")
|
|
|
|
summary = summarise(collect(store))
|
|
assert summary["haiku"]["total"] == 2
|
|
assert summary["haiku"]["good"] == 1
|
|
assert summary["haiku"][UNMARKED] == 1
|
|
assert summary["opus"]["total"] == 1
|