import io from pathlib import Path from typing import Any import pytest from nisse_core.cancel import STOPPED_RESULT, Cancel from nisse_core.canonical import ( AssistantText, CanonicalEvent, DelegationCompleted, DelegationStarted, StoredEvent, ToolCall, ToolResult, ) from nisse_core.execution import SSHHost from nisse_core.llm import BudgetExhausted, LLMError, UpstreamUnavailable from nisse_core.oplog import OpLog from nisse_core.state import Transcript from nisse_dev.delegation import ( DELEGATE_TOOLS, DELEGATION_INTENT, Caps, Delegation, DelegationResult, delegation_reconciler, ) from nisse_dev.git import GitRepo from nisse_dev.state import StateStore from nisse_dev.tools import Workspace class ScriptedClient: def __init__(self, turns: list[list[CanonicalEvent]]) -> None: self._turns = turns self.calls: list[dict[str, Any]] = [] def complete( self, events: list[StoredEvent], tools: list[dict[str, Any]] | None, model: str, system: str | None = None, ) -> list[CanonicalEvent]: self.calls.append({"tools": tools, "model": model, "system": system}) if not self._turns: return [AssistantText(text="still thinking", model=model)] return self._turns.pop(0) class FailingClient: def complete(self, *args: Any, **kwargs: Any) -> list[CanonicalEvent]: raise LLMError("frontier unavailable") def call(tool: str, cid: str = "c1", **arguments: Any) -> ToolCall: return ToolCall(call_id=cid, tool=tool, arguments=arguments) def hand_back(succeeded: bool = True, cid: str = "hb", **overrides: Any) -> ToolCall: args = { "changed": "added greet.py", "now_true": "greet() returns a greeting and its test passes", "next": "open the PR", "succeeded": succeeded, } args.update(overrides) return ToolCall(call_id=cid, tool="hand_back", arguments=args) @pytest.fixture def store(tmp_path: Path) -> StateStore: return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) @pytest.fixture def repo(ssh_host: SSHHost, tmp_path: Path) -> GitRepo: r = GitRepo(ssh_host, tmp_path / "work") r.init() r.configure_identity("bot", "bot@example.test") (r.root / "seed.txt").write_text("seed") r.commit_all("seed") return r @pytest.fixture def ws(repo: GitRepo) -> Workspace: return Workspace(repo.host, repo.root, timeout_s=10) ITEM = "t" # the work item these delegations run in def log(store: StateStore) -> Transcript: return store.transcript(ITEM, "helper") class Bound: """A delegation run in the test's work item, as the executor runs it.""" def __init__(self, delegation: Delegation, store: StateStore) -> None: self.delegation = delegation self.store = store def run(self, **kw: Any) -> DelegationResult: return self.delegation.run(log(self.store), **kw) def resume(self, intent: Any) -> DelegationResult: return self.delegation.resume(intent) def recover(self, notify: Any) -> DelegationResult | None: return self.delegation.recover(notify) def build(client: Any, store: StateStore, ws: Workspace, repo: GitRepo, **kw: Any): return Bound( Delegation(client, store, ws, repo, model="claude-haiku-4-5", **kw), store ) def test_successful_delegation_returns_a_structured_handback( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: client = ScriptedClient( [ [call("write_file", path="greet.py", content="def greet(): ...")], [hand_back()], ] ) result = build(client, store, ws, repo).run(brief="add greet", branch="d1") assert result.usable assert result.handback is not None assert "greet.py" in result.handback.changed assert "what to do next" in result.handback.render().lower() assert result.artifacts == ["greet.py"] def test_delegation_works_on_its_own_branch( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: client = ScriptedClient( [[call("write_file", path="a.py", content="x")], [hand_back()]] ) build(client, store, ws, repo).run(brief="b", branch="deleg-1") assert repo.current_branch() == "deleg-1" def test_abort_discards_the_branch_and_its_work( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # L9: abort must leave nothing to resume on top of. client = ScriptedClient( [ [call("write_file", path="half-done.py", content="oops")], [hand_back(succeeded=False, changed="could not finish")], ] ) result = build(client, store, ws, repo).run(brief="b", branch="deleg-2") assert not result.usable assert repo.current_branch() == "main" assert not (repo.root / "half-done.py").exists() def test_stop_discards_the_branch_and_settles_the_intent( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # L2's stop is treated like a failure, not like an outage: the operator # stopped it for a reason, so nothing is kept for resuming. cancel = Cancel() client = ScriptedClient( [ [call("write_file", path="half-done.py", content="oops")], [call("list_dir", path=".")], ] ) original = client.complete def complete_then_stop(*args: Any, **kwargs: Any) -> list[CanonicalEvent]: produced = original(*args, **kwargs) if len(client.calls) == 2: cancel.set() # the operator says stop during turn 2's model call return produced client.complete = complete_then_stop # type: ignore[method-assign] result = build(client, store, ws, repo, cancel=cancel).run( brief="b", branch="deleg-stop" ) assert result.status == "stopped" assert not result.usable assert repo.current_branch() == "main" assert not repo.branch_exists("deleg-stop") assert not (repo.root / "half-done.py").exists() assert store.pending_intents() == [] # The unrun call still has a result, so the transcript renders. skipped = [ e.event for e in log(store).read() if isinstance(e.event, ToolResult) and e.event.result == STOPPED_RESULT ] assert len(skipped) == 1 def test_turn_cap_is_hard(store: StateStore, ws: Workspace, repo: GitRepo) -> None: forever = [[call("list_dir", path=".")] for _ in range(20)] client = ScriptedClient(forever) result = build(client, store, ws, repo, caps=Caps(turns=3)).run( brief="b", branch="d" ) assert result.status == "capped" assert "turn cap of 3" in result.note assert len(client.calls) == 3 def test_wall_clock_cap_is_hard( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: ticks = iter([0.0, 1.0, 99.0, 99.0, 99.0]) client = ScriptedClient([[call("list_dir", path=".")] for _ in range(10)]) result = build( client, store, ws, repo, caps=Caps(turns=10, wall_clock_s=60.0), now=lambda: next(ticks), ).run(brief="b", branch="d") assert result.status == "capped" assert "wall-clock cap" in result.note def test_cap_breach_is_not_usable_and_discards_work( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: client = ScriptedClient([[call("write_file", path="partial.py", content="x")]] * 5) result = build(client, store, ws, repo, caps=Caps(turns=2)).run( brief="b", branch="d" ) assert not result.usable # the caller must gate, not resume (L9) assert not (repo.root / "partial.py").exists() def test_events_are_checkpointed_as_they_arrive( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: client = ScriptedClient( [[call("write_file", path="a.py", content="x")], [hand_back()]] ) build(client, store, ws, repo).run(brief="add a", branch="d") kinds = [type(e.event).__name__ for e in log(store).read()] assert kinds == [ "DelegationStarted", "ToolCall", "ToolResult", "ToolCall", # hand_back "ToolResult", "DelegationCompleted", ] def test_composite_events_record_the_brief_caps_and_outcome( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: client = ScriptedClient( [[call("write_file", path="a.py", content="x")], [hand_back()]] ) caps = Caps(turns=7, tokens=1234, wall_clock_s=42.0) build(client, store, ws, repo, caps=caps).run(brief="the brief", branch="d") events = [e.event for e in log(store).read()] started = next(e for e in events if isinstance(e, DelegationStarted)) completed = next(e for e in events if isinstance(e, DelegationCompleted)) assert started.brief == "the brief" assert started.model == "claude-haiku-4-5" assert started.caps == {"turns": 7, "tokens": 1234, "wall_clock_s": 42.0} # L9: this pair is the eval record, so it must carry the outcome. assert completed.exit_status == "handback" assert "greet()" in completed.handback def test_hand_back_tool_is_offered_alongside_the_executor_tools() -> None: names = [t["function"]["name"] for t in DELEGATE_TOOLS] assert names[-1] == "hand_back" assert "run_command" in names # full tool access, not a consult (L9) def test_malformed_handback_is_rejected_and_the_delegation_continues( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: client = ScriptedClient( [ [hand_back(cid="bad", changed=" ")], # blank field [call("write_file", path="a.py", content="x")], [hand_back(cid="good")], ] ) result = build(client, store, ws, repo).run(brief="b", branch="d") assert result.usable # recovered rather than accepting a useless note assert len(client.calls) == 3 def test_non_boolean_success_flag_is_rejected( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: client = ScriptedClient( [ [call("write_file", path="a.py", content="x")], [hand_back(succeeded="yes")], # not a bool: unparseable [hand_back(cid="ok")], ] ) result = build(client, store, ws, repo).run(brief="b", branch="d") assert result.usable assert len(client.calls) == 3 def test_client_failure_is_a_defined_return_not_a_crash( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: result = build(FailingClient(), store, ws, repo).run(brief="b", branch="d") assert result.status == "failed" assert "frontier unavailable" in result.note assert not result.usable def test_a_delegate_that_never_hands_back_is_capped_not_believed( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # Prose forever: the delegate never reports, so its tree state is unknown. client = ScriptedClient([]) result = build(client, store, ws, repo, caps=Caps(turns=3)).run( brief="b", branch="d" ) assert result.status == "capped" assert not result.usable def test_success_claimed_without_any_change_is_refused( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # The handback plus file state is the whole interface back, so a note # describing work that was never done is worse than no note. client = ScriptedClient( [ [hand_back(cid="early")], [call("write_file", path="real.py", content="x")], [hand_back(cid="earned")], ] ) result = build(client, store, ws, repo).run(brief="b", branch="d") assert result.usable assert len(client.calls) == 3 # the first claim was sent back assert (repo.root / "real.py").exists() def test_failure_may_be_reported_without_doing_work( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # "This sub-task is impossible" is a legitimate finding that requires no # changes; only a claim of success needs evidence. client = ScriptedClient( [[hand_back(succeeded=False, changed="nothing: the API does not exist")]] ) result = build(client, store, ws, repo).run(brief="b", branch="d") assert result.status == "failed" assert not result.usable assert len(client.calls) == 1 # believed immediately, not challenged # --- L8 boot step 5: the resume rule ----------------------------------------- def pending_delegation_intent(store: StateStore, branch: str = "d") -> Any: store.begin_intent( DELEGATION_INTENT, {"branch": branch, "base": "main", "brief": "finish it"}, cause="test", work_item=ITEM, # owned by the work item it ran in ) return store.pending_intents()[-1] def test_intent_is_recorded_and_settled_by_a_completed_delegation( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: client = ScriptedClient( [[call("write_file", path="a.py", content="x")], [hand_back()]] ) build(client, store, ws, repo).run(brief="b", branch="d") # Nothing left in flight: an aborted delegation is a finished one too. assert store.pending_intents() == [] def test_aborted_delegation_also_settles_its_intent( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: client = ScriptedClient([[hand_back(succeeded=False, changed="cannot")]]) build(client, store, ws, repo).run(brief="b", branch="d") assert store.pending_intents() == [] def test_clean_cut_after_a_tool_result_is_resumable( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: repo.checkout_new_branch("d") (repo.root / "partial.py").write_text("x") repo.commit_all("partial work") log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={})) log(store).append(ToolCall(call_id="c", tool="write_file", arguments={})) log(store).append(ToolResult(call_id="c", result="wrote", success=True)) intent = pending_delegation_intent(store) assert delegation_reconciler(store, repo)(intent) is True def test_dangling_tool_call_gates( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # The effect of the call is unknown and not verifiable by inspection. repo.checkout_new_branch("d") log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={})) log(store).append(ToolCall(call_id="c", tool="run_command", arguments={})) intent = pending_delegation_intent(store) assert delegation_reconciler(store, repo)(intent) is None def test_missing_branch_means_nothing_started( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={})) log(store).append(ToolResult(call_id="c", result="ok", success=True)) intent = pending_delegation_intent(store, branch="never-created") assert delegation_reconciler(store, repo)(intent) is False def test_a_finished_delegation_settles_rather_than_resumes( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: repo.checkout_new_branch("d") log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={})) log(store).append( DelegationCompleted( handback="done", artifacts=[], tokens=0, duration_s=1.0, exit_status="handback", ) ) intent = pending_delegation_intent(store) assert delegation_reconciler(store, repo)(intent) is True def test_resume_continues_the_same_delegation( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # Interrupt after one turn, then resume: no second delegation_started, # because L7 defines a session as the delegation, not the process. repo.checkout_new_branch("d") log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={})) log(store).append(ToolResult(call_id="c", result="wrote a.py", success=True)) intent = pending_delegation_intent(store) client = ScriptedClient( [[call("write_file", path="b.py", content="x")], [hand_back()]] ) result = build(client, store, ws, repo).resume(intent) assert result.usable starts = [e for e in log(store).read() if isinstance(e.event, DelegationStarted)] assert len(starts) == 1 assert store.pending_intents() == [] assert (repo.root / "b.py").exists() def test_resume_onto_a_missing_branch_fails_cleanly( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: intent = pending_delegation_intent(store, branch="gone") result = build(ScriptedClient([]), store, ws, repo).resume(intent) assert result.status == "failed" assert not result.usable assert store.pending_intents() == [] # not left dangling for the next boot # --- boot recovery sweep ------------------------------------------------ def test_recover_resumes_a_clean_cut_with_a_notice( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: repo.checkout_new_branch("d") log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={})) log(store).append(ToolResult(call_id="c", result="wrote a.py", success=True)) pending_delegation_intent(store) notices: list[str] = [] client = ScriptedClient( [[call("write_file", path="b.py", content="x")], [hand_back()]] ) result = build(client, store, ws, repo).recover(notices.append) assert result is not None and result.usable # L8: a notice, not a question — a clean cut needs no operator decision. assert notices and "resuming the delegation" in notices[0] def test_recover_settles_a_finished_delegation_without_resuming( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: repo.checkout_new_branch("d") log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={})) log(store).append( DelegationCompleted( handback="done", artifacts=[], tokens=0, duration_s=1.0, exit_status="handback", ) ) pending_delegation_intent(store) client = ScriptedClient([]) assert build(client, store, ws, repo).recover(lambda _m: None) is None assert store.pending_intents() == [] assert client.calls == [] # the model was never asked def test_recover_discards_an_intent_whose_branch_never_existed( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: log(store).append(ToolResult(call_id="c", result="ok", success=True)) pending_delegation_intent(store, branch="never-made") assert build(ScriptedClient([]), store, ws, repo).recover(lambda _m: None) is None assert store.pending_intents() == [] def test_recover_leaves_an_unclean_cut_for_the_boot_gate( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: repo.checkout_new_branch("d") log(store).append(DelegationStarted(brief="b", role="helper", model="m", caps={})) log(store).append(ToolCall(call_id="c", tool="run_command", arguments={})) pending_delegation_intent(store) assert build(ScriptedClient([]), store, ws, repo).recover(lambda _m: None) is None # Still pending, so reconcile_on_boot gates rather than blind-resuming. assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT] # --- the token cap, which L9 calls hard and was decoration ------------- class MeteredClient(ScriptedClient): """A client that reports usage, as the real ones do.""" def __init__(self, turns: list, per_call: int = 1000) -> None: super().__init__(turns) self._per_call = per_call self.last_usage: dict[str, int] = {} def complete(self, *args: Any, **kwargs: Any): produced = super().complete(*args, **kwargs) self.last_usage = {"input": self._per_call, "output": self._per_call} return produced def test_token_cap_stops_a_delegation( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # An unbounded delegation can exhaust a daily budget and take external # models away until it resets, so the cap has to be real. client = MeteredClient( [[call("list_dir", path=".")] for _ in range(20)], per_call=600 ) result = build(client, store, ws, repo, caps=Caps(turns=20, tokens=2000)).run( brief="b", branch="d" ) assert result.status == "capped" assert "token cap of 2000" in result.note # 600 in + 600 out per call: stops on the second turn, not the twentieth. assert len(client.calls) == 2 def test_tokens_spent_are_reported_not_left_at_zero( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: client = MeteredClient( [[call("write_file", path="a.py", content="x")], [hand_back()]], per_call=500 ) result = build(client, store, ws, repo).run(brief="b", branch="d") assert result.usable assert result.tokens == 2000 # two calls, 1000 each completed = next( e.event for e in log(store).read() if isinstance(e.event, DelegationCompleted) ) # L9 evaluates cost against outcome, which needs a real number. assert completed.tokens == 2000 def test_a_client_without_usage_still_works( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # Scripted and third-party clients need not report usage; the cap simply # does not bind rather than crashing the delegation. client = ScriptedClient( [[call("write_file", path="a.py", content="x")], [hand_back()]] ) assert build(client, store, ws, repo).run(brief="b", branch="d").usable def test_budget_exhaustion_preserves_the_work_for_later( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: """An outage is not a bad delegate. The branch may hold perfectly good work, so it is kept and the intent left pending — the next boot treats it exactly like a crash mid-delegation and resumes if the cut was clean. Discarding would throw away work because the budget ran out. """ class BrokeClient(ScriptedClient): def complete(self, *args: Any, **kwargs: Any): if self.calls: raise BudgetExhausted("daily budget exhausted") return super().complete(*args, **kwargs) client = BrokeClient([[call("write_file", path="progress.py", content="real")]]) result = build(client, store, ws, repo).run(brief="b", branch="d") assert result.status == "interrupted" assert result.interrupted and not result.usable # The work survives, on its branch. assert repo.current_branch() == "d" assert (repo.root / "progress.py").exists() # And the intent stays pending, which is what L8's resume rule keys on. assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT] def test_an_ordinary_failure_still_discards( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # Only external interruptions are preserved; a delegate that made a mess # is still cleaned up (L9). client = ScriptedClient([[hand_back(succeeded=False, changed="cannot")]]) result = build(client, store, ws, repo).run(brief="b", branch="d") assert result.status == "failed" assert not result.interrupted assert repo.current_branch() == "main" assert store.pending_intents() == [] def test_a_network_outage_also_preserves_the_work( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # Same reasoning as an exhausted budget: the branch may be fine, and the # outage says nothing about the quality of the work on it. class FlakyClient(ScriptedClient): def complete(self, *args: Any, **kwargs: Any): if self.calls: raise UpstreamUnavailable("LLM request failed: connection refused") return super().complete(*args, **kwargs) client = FlakyClient([[call("write_file", path="progress.py", content="real")]]) result = build(client, store, ws, repo).run(brief="b", branch="d") assert result.status == "interrupted" assert (repo.root / "progress.py").exists() assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT] def test_a_bad_request_is_not_given_the_benefit_of_the_doubt( store: StateStore, ws: Workspace, repo: GitRepo ) -> None: # A 4xx that is not a rate limit means we sent something wrong; treating # that as an outage would preserve branches and hide a real bug. class BadRequestClient(ScriptedClient): def complete(self, *args: Any, **kwargs: Any): raise LLMError("LLM request returned 400: malformed tool schema") result = build(BadRequestClient([]), store, ws, repo).run(brief="b", branch="d") assert result.status == "failed" assert not result.interrupted