"""The orchestrator's resting-state invariant, checked under injected faults. Working states (planning, executing) are only right while work runs. After any message is handled the agent must be somewhere it can wait for the operator, and some reply must lead on from there. The first live run found four ways to break this one at a time; this drives a whole project with a fault at each step instead. """ import io from pathlib import Path from typing import Any import pytest from nisse_dev.control import ( AWAIT_ARCH_SIGNOFF, AWAIT_INPUT, AWAIT_MILESTONE_SIGNOFF, EXECUTING, IDLE, MILESTONE_PLANNING, PLANNING_ARCH, PR_OPEN, AskHuman, Orchestrator, PlanningError, SubmitPlan, ) from nisse_dev.executor import ExecutorOutcome from nisse_dev.gitea import GiteaError, PullRequest from nisse_dev.oplog import OpLog from nisse_dev.publish import PublishResult from nisse_dev.state import StateStore from nisse_dev.transport import InMemoryTransport WORKING = {PLANNING_ARCH, MILESTONE_PLANNING, EXECUTING} class Faults: """One fault, fired once, at the nth call to one site.""" def __init__(self, site: str, nth: int, fault: Any) -> None: self._site, self._nth, self._fault = site, nth, fault self._calls: dict[str, int] = {} self.fired = False def check(self, site: str) -> Any: """Raise or return the fault if this call is the one; else None.""" self._calls[site] = self._calls.get(site, 0) + 1 if self.fired or site != self._site or self._calls[site] != self._nth: return None self.fired = True if isinstance(self._fault, Exception): raise self._fault return self._fault class Planner: def __init__(self, faults: Faults) -> None: self._faults = faults def plan( self, transcript: object, gate: str, project: str, feedback: str | None = None, answer: str | None = None, ) -> SubmitPlan | AskHuman: return self._faults.check("planner") or SubmitPlan(f"{gate} plan") class Executor: def __init__(self, faults: Faults) -> None: self._faults = faults def run(self, transcript: object, milestone: int) -> ExecutorOutcome: return self._faults.check("executor") or ExecutorOutcome("completed", "done") class Publisher: def __init__(self, faults: Faults) -> None: self._faults = faults self._opened = 0 def reconcilers(self) -> dict[str, Any]: return {} def publish(self, **kwargs: Any) -> PublishResult: self._faults.check("publisher") self._opened += 1 pr = PullRequest(number=self._opened, state="open", head="h", url="u") return PublishResult(pr, f"opened PR #{self._opened}") def next_reply(store: StateStore, used_project: bool) -> str: """What an operator following the prompts would say next.""" state = store.get_state() assert state is not None data = state.data match state.state: case "IDLE": return "status" if used_project else "use demo" case "AWAIT_ARCH_SIGNOFF": return "approve" case "AWAIT_MILESTONE_SIGNOFF": if data["gate"]["approved"]: return f"start milestone {data['milestone']}" return "approve" case "PR_OPEN": return "next milestone" if data["milestone"] < 2 else "project complete" case "AWAIT_INPUT": return "continue" raise AssertionError(f"stranded in {state.state}") FAULTS = [ ("planner", 1, RuntimeError("bug in planning")), ("planner", 1, PlanningError("planner returned 400")), ("planner", 1, AskHuman("which language?")), ("planner", 2, RuntimeError("bug in planning")), ("planner", 3, OSError("disk full")), ("executor", 1, RuntimeError("bug in the executor")), ("executor", 1, ExecutorOutcome("escalate", "stuck")), ("executor", 1, ExecutorOutcome("capped", "max turns")), ("executor", 2, ExecutorOutcome("stopped", "operator")), ("publisher", 1, RuntimeError("bug in publishing")), ("publisher", 1, GiteaError("gitea is down")), ("publisher", 2, RuntimeError("bug in publishing")), ] @pytest.mark.parametrize( ("site", "nth", "fault"), FAULTS, ids=[f"{s}{n}-{f!r}" for s, n, f in FAULTS] ) def test_a_fault_anywhere_never_strands_the_agent( tmp_path: Path, site: str, nth: int, fault: Any ) -> None: faults = Faults(site, nth, fault) store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) transport = InMemoryTransport() Orchestrator( store, transport, Planner(faults), operators={"don"}, executor=Executor(faults), publisher=Publisher(faults), ).start() used_project = False for _ in range(40): reply = next_reply(store, used_project) used_project = used_project or reply.startswith("use ") transport.receive("don", reply) state = store.get_state() assert state is not None assert state.state not in WORKING, ( f"after {reply!r}: stranded in {state.state}; " f"last said: {transport.outbound[-1].text!r}" ) if reply == "project complete": break else: raise AssertionError("never reached project complete") assert faults.fired assert store.get_state().state == IDLE def test_a_signoff_state_without_its_gate_offers_a_way_out(tmp_path: Path) -> None: faults = Faults("none", 0, None) store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) transport = InMemoryTransport() orch = Orchestrator(store, transport, Planner(faults), operators={"don"}) orch.start() transport.receive("don", "use demo") state = store.get_state() store.transition( AWAIT_ARCH_SIGNOFF, cause="test", project="demo", data={**state.data, "gate": None}, ) orch._data["gate"] = None # as a restart would load it transport.receive("don", "approve") assert store.get_state().state == AWAIT_INPUT transport.receive("don", "continue") assert store.get_state().state == AWAIT_ARCH_SIGNOFF class RaisingReviewExecutor: def run(self, transcript: object, milestone: int) -> ExecutorOutcome: raise RuntimeError("bug while addressing review") def test_a_failing_review_response_keeps_pr_open_and_says_so( tmp_path: Path, ) -> None: faults = Faults("none", 0, None) store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) transport = InMemoryTransport() orch = Orchestrator( store, transport, Planner(faults), operators={"don"}, gitea_operators={"don"}, executor=Executor(faults), publisher=Publisher(faults), ) orch.start() for reply in ("use demo", "approve", "approve", "start milestone 1"): transport.receive("don", reply) assert store.get_state().state == PR_OPEN orch._executor = RaisingReviewExecutor() class Comment: id, author, body = 1, "don", "add a test" orch.handle_review_comment(Comment()) assert store.get_state().state == PR_OPEN assert "could not handle a review comment" in transport.outbound[-1].text transport.receive("don", "project complete") assert store.get_state().state == IDLE def test_the_stub_configuration_reaches_every_state(tmp_path: Path) -> None: store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) transport = InMemoryTransport() Orchestrator( store, transport, Planner(Faults("none", 0, None)), operators={"don"} ).start() for reply in ("use demo", "approve", "approve", "start milestone 1"): transport.receive("don", reply) assert store.get_state().state == PR_OPEN transport.receive("don", "next milestone") assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF