dongarandClaude Opus 5.5
1ad0b2147d
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m22s
Development Plan M0.1. Every section reference in code, tests, specs/, the example config, the tuning guide, and the CLAUDE.md and README files now cites docs/Architecture.md (P, L, D, O, X, Q) instead of the old spec's A-H, and docs/Architecture-Build-Spec.md is deleted (git keeps it). The plan's old-to-new table goes with it. Mostly mechanical, with a per-site choice where one old section split: A4 became L2 where it meant stop/abandon/the loop and L3 where it meant approvals; A10 became L6 in the context-budget code and L7 elsewhere; B2 became D4 for outbound Gitea and publishing and D5 for review intake; 'A3 step N' became the matching step of L8's boot sequence. Comments describing behaviour the architecture drops (planning always escalating, thresholds tuned at an old build step) now describe the current code and point at the Development Plan. docs/Architecture.md gains one line the rewrite missed: intake outages delay but never fail work (old B2), now in L5, where the poller and Matrix code cite it. Comments and docs only; no behaviour change. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
130 lines
3.9 KiB
Python
130 lines
3.9 KiB
Python
import io
|
|
import json
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from nisse_dev.canonical import (
|
|
AssistantText,
|
|
DelegationCompleted,
|
|
DelegationStarted,
|
|
)
|
|
from nisse_dev.evals import UNMARKED, collect, summarise, to_jsonl
|
|
from nisse_dev.oplog import OpLog
|
|
from nisse_dev.state import StateStore
|
|
|
|
|
|
@pytest.fixture
|
|
def store(tmp_path: Path) -> StateStore:
|
|
return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
|
|
|
|
|
def record_pair(
|
|
store: StateStore,
|
|
*,
|
|
brief: str = "add greet",
|
|
model: str = "claude-haiku-4-5",
|
|
role: str = "delegate",
|
|
status: str = "handback",
|
|
) -> int:
|
|
store.append_event(
|
|
DelegationStarted(brief=brief, role=role, model=model, caps={"turns": 5})
|
|
)
|
|
store.append_event(
|
|
DelegationCompleted(
|
|
handback="what changed: greet.py",
|
|
artifacts=["greet.py"],
|
|
tokens=1200,
|
|
duration_s=9.5,
|
|
exit_status=status,
|
|
)
|
|
)
|
|
return store.read_events()[-1].id
|
|
|
|
|
|
def test_pairs_a_start_with_its_completion(store: StateStore) -> None:
|
|
record_pair(store)
|
|
[record] = collect(store)
|
|
assert record.brief == "add greet"
|
|
assert record.model == "claude-haiku-4-5"
|
|
assert record.exit_status == "handback"
|
|
assert record.artifacts == ["greet.py"]
|
|
assert record.verdict == UNMARKED
|
|
|
|
|
|
def test_unpaired_start_is_skipped(store: StateStore) -> None:
|
|
# A crash mid-delegation leaves a start with no completion; a half-formed
|
|
# record would pollute the set L9 exists to make trustworthy.
|
|
store.append_event(
|
|
DelegationStarted(brief="died", role="delegate", model="m", caps={})
|
|
)
|
|
assert collect(store) == []
|
|
|
|
|
|
def test_unrelated_events_between_the_pair_are_ignored(store: StateStore) -> None:
|
|
store.append_event(
|
|
DelegationStarted(brief="b", role="delegate", model="m", caps={})
|
|
)
|
|
store.append_event(AssistantText(text="working", model="m"))
|
|
store.append_event(
|
|
DelegationCompleted(
|
|
handback="done",
|
|
artifacts=[],
|
|
tokens=1,
|
|
duration_s=1.0,
|
|
exit_status="handback",
|
|
)
|
|
)
|
|
assert len(collect(store)) == 1
|
|
|
|
|
|
def test_verdicts_attach_to_the_completion(store: StateStore) -> None:
|
|
event_id = record_pair(store)
|
|
store.mark_outcome(event_id, "good", "clean handback, tests passed")
|
|
[record] = collect(store)
|
|
assert record.verdict == "good"
|
|
assert "tests passed" in record.note
|
|
|
|
|
|
def test_a_verdict_can_be_revised(store: StateStore) -> None:
|
|
# Judgements get revised; the log they describe does not.
|
|
event_id = record_pair(store)
|
|
store.mark_outcome(event_id, "good")
|
|
store.mark_outcome(event_id, "bad", "regressed later")
|
|
[record] = collect(store)
|
|
assert record.verdict == "bad"
|
|
assert record.note == "regressed later"
|
|
|
|
|
|
def test_consults_and_delegations_share_the_set(store: StateStore) -> None:
|
|
record_pair(store, role="delegate")
|
|
record_pair(store, role="consultant", brief="which approach?")
|
|
roles = [r.role for r in collect(store)]
|
|
assert roles == ["delegate", "consultant"]
|
|
|
|
|
|
def test_collect_can_start_after_a_marker(store: StateStore) -> None:
|
|
record_pair(store, brief="old")
|
|
marker = store.read_events()[-1].id
|
|
record_pair(store, brief="new")
|
|
assert [r.brief for r in collect(store, after=marker)] == ["new"]
|
|
|
|
|
|
def test_jsonl_round_trips(store: StateStore) -> None:
|
|
record_pair(store)
|
|
line = to_jsonl(collect(store))
|
|
assert json.loads(line)["model"] == "claude-haiku-4-5"
|
|
|
|
|
|
def test_summary_groups_by_model(store: StateStore) -> None:
|
|
good = record_pair(store, model="haiku")
|
|
record_pair(store, model="haiku", status="capped")
|
|
record_pair(store, model="opus")
|
|
store.mark_outcome(good, "good")
|
|
|
|
summary = summarise(collect(store))
|
|
assert summary["haiku"]["total"] == 2
|
|
assert summary["haiku"]["good"] == 1
|
|
assert summary["haiku"][UNMARKED] == 1
|
|
assert summary["opus"]["total"] == 1
|