import io from pathlib import Path from typing import Any import pytest from nisse_dev.canonical import ( AssistantText, CanonicalEvent, PlanSubmitted, StoredEvent, ToolCall, UserMessage, ) from nisse_dev.control import ( AWAIT_ARCH_SIGNOFF, AWAIT_MILESTONE_SIGNOFF, AskHuman, Orchestrator, PlanningError, SubmitPlan, ) from nisse_dev.llm import BudgetExhausted, RoleMap from nisse_dev.oplog import OpLog from nisse_dev.planner import PLANNER_TOOLS, LLMPlanningAgent from nisse_dev.state import StateStore, Transcript from nisse_dev.transport import InMemoryTransport ROLES = RoleMap( executor="local/qwen", planner="anthropic/claude-opus-5", helper="anthropic/claude-opus-5", expert="anthropic/claude-opus-5", ) class ScriptedClient: """Returns canned canonical events per call, recording what it was asked.""" def __init__(self, turns: list[list[CanonicalEvent]]) -> None: self._turns = turns self.calls: list[dict[str, Any]] = [] def complete( self, events: list[StoredEvent], tools: list[dict[str, Any]] | None, model: str, system: str | None = None, ) -> list[CanonicalEvent]: self.calls.append( {"events": events, "tools": tools, "model": model, "system": system} ) if not self._turns: return [] return self._turns.pop(0) def plan_call(content: str, gate: str = "arch") -> ToolCall: return ToolCall( call_id="call-1", tool="submit_plan", arguments={"type": gate, "content": content}, ) ITEM = "t" # the work item the unit tests plan in def log(store: StateStore) -> Transcript: return store.transcript(ITEM, "planner") @pytest.fixture def store(tmp_path: Path) -> StateStore: return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) def test_submit_plan_tool_call_becomes_submit_plan(store: StateStore) -> None: client = ScriptedClient([[plan_call("step 1: create src/app.py")]]) agent = LLMPlanningAgent(client, ROLES) action = agent.plan(log(store), "arch", "demo") assert action == SubmitPlan("step 1: create src/app.py") def test_ask_human_tool_call_becomes_ask_human(store: StateStore) -> None: client = ScriptedClient( [ [ ToolCall( call_id="call-1", tool="ask_human", arguments={"question": "postgres or sqlite?"}, ) ] ] ) action = LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") assert action == AskHuman("postgres or sqlite?") def test_planner_role_and_tools_are_used(store: StateStore) -> None: client = ScriptedClient([[plan_call("plan")]]) LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") call = client.calls[0] assert call["model"] == "anthropic/claude-opus-5" # never the executor assert [t["function"]["name"] for t in call["tools"]] == [ "submit_plan", "ask_human", ] assert "demo" in call["system"] and "arch" in call["system"] def test_no_proceed_tool_is_offered() -> None: names = {tool["function"]["name"] for tool in PLANNER_TOOLS} assert names == {"submit_plan", "ask_human"} # no self-advancement path def test_model_output_is_appended_to_the_log(store: StateStore) -> None: client = ScriptedClient( [[AssistantText(text="thinking out loud", model="m"), plan_call("the plan")]] ) LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") kinds = [type(e.event).__name__ for e in log(store).read()] assert kinds == ["AssistantText", "ToolCall", "ToolResult"] def test_gate_tool_call_is_answered_in_the_log(store: StateStore) -> None: # submit_plan ends the turn and has no natural result, but a tool call # left dangling makes the next render invalid for the provider. client = ScriptedClient([[plan_call("the plan")]]) LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") events = [e.event for e in log(store).read()] assert [type(e).__name__ for e in events] == ["ToolCall", "ToolResult"] assert events[1].call_id == events[0].call_id assert "posted for approval" in events[1].result def test_prose_only_turn_retries_once_then_raises(store: StateStore) -> None: client = ScriptedClient( [ [AssistantText(text="here's what I think", model="m")], [AssistantText(text="still prose", model="m")], ] ) with pytest.raises(PlanningError, match="no submit_plan or ask_human"): LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") assert len(client.calls) == 2 assert "did not call a tool" in client.calls[1]["system"] def test_retry_succeeds_after_prose(store: StateStore) -> None: client = ScriptedClient( [[AssistantText(text="prose", model="m")], [plan_call("the real plan")]] ) action = LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") assert action == SubmitPlan("the real plan") def test_malformed_tool_arguments_are_rejected(store: StateStore) -> None: blank = ToolCall(call_id="c", tool="submit_plan", arguments={"content": " "}) client = ScriptedClient([[blank], [blank]]) with pytest.raises(PlanningError): LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") def test_planner_sees_prior_transcript(store: StateStore) -> None: store.transcript(ITEM, "operator").append( UserMessage(text="use demo", source="transport:@don") ) client = ScriptedClient([[plan_call("plan")]]) LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") seen = [getattr(e.event, "text", None) for e in client.calls[0]["events"]] assert "[operator, Matrix] use demo" in seen # --- orchestrator integration (still no network) ------------------------- def build(tmp_path: Path, client: ScriptedClient) -> tuple[Orchestrator, StateStore]: store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) transport = InMemoryTransport() agent = LLMPlanningAgent(client, ROLES) orch = Orchestrator(store, transport, agent, operators={"don"}, roles=ROLES) orch.start() return orch, store, transport def test_use_project_produces_llm_plan_and_transcript(tmp_path: Path) -> None: client = ScriptedClient([[plan_call("1. create app.py\n2. add a test")]]) _, store, transport = build(tmp_path, client) transport.receive("don", "use demo") request = transport.outbound[-1] assert request.kind == "approval_request" assert "create app.py" in request.text # The work item's own events: `use demo` was said while idle, and is # carried into the item it opened (D1). item = store.active_item() events = [e.event for e in store.transcript(item.id, "operator").read()] kinds = [type(e).__name__ for e in events] assert kinds == [ "UserMessage", "Fact", "ToolCall", "ToolResult", "PlanSubmitted", "GateOpened", ] submitted = next(e for e in events if isinstance(e, PlanSubmitted)) assert submitted.model == "anthropic/claude-opus-5" # pinned planner def test_roles_pinned_at_task_start(tmp_path: Path) -> None: client = ScriptedClient([[plan_call("plan")]]) _, store, transport = build(tmp_path, client) transport.receive("don", "use demo") state = store.get_state() assert state is not None assert state.data["roles"]["planner"] == "anthropic/claude-opus-5" def test_planning_failure_gates_to_await_input(tmp_path: Path) -> None: prose = [AssistantText(text="prose", model="m")] client = ScriptedClient([prose, prose]) _, store, transport = build(tmp_path, client) transport.receive("don", "use demo") state = store.get_state() assert state is not None assert state.state == "AWAIT_INPUT" assert transport.outbound[-1].kind == "question" def test_operator_can_recover_from_a_planning_failure(tmp_path: Path) -> None: # Regression: AWAIT_INPUT reached by any path other than ask_human used to # record no pending item, so every operator reply was refused with "no # pending question" and the agent was stuck for good. prose = [AssistantText(text="prose", model="m")] client = ScriptedClient([prose, prose, [plan_call("recovered plan")]]) _, store, transport = build(tmp_path, client) transport.receive("don", "use demo") assert store.get_state().state == "AWAIT_INPUT" transport.receive("don", "try again, keep it short") assert store.get_state().state == AWAIT_ARCH_SIGNOFF assert "recovered plan" in transport.outbound[-1].text def test_operator_can_recover_after_a_stopped_milestone(tmp_path: Path) -> None: class StoppedExecutor: def run(self, transcript: object, milestone: int) -> Any: from nisse_dev.executor import ExecutorOutcome return ExecutorOutcome("escalate", "tests failed twice") store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) transport = InMemoryTransport() client = ScriptedClient([[plan_call("p1")], [plan_call("p2")]]) orch = Orchestrator( store, transport, LLMPlanningAgent(client, ROLES), operators={"don"}, roles=ROLES, executor=StoppedExecutor(), ) orch.start() transport.receive("don", "use demo") transport.receive("don", "approve") transport.receive("don", "approve") transport.receive("don", "start milestone 1") assert store.get_state().state == "AWAIT_INPUT" transport.receive("don", "ok, I fixed the dependency") assert store.get_state().state == AWAIT_MILESTONE_SIGNOFF assert "start milestone 1" in transport.outbound[-1].text def test_rejection_feedback_reaches_the_planner_via_the_log( tmp_path: Path, ) -> None: client = ScriptedClient([[plan_call("v1")], [plan_call("v2")]]) _, _store, transport = build(tmp_path, client) transport.receive("don", "use demo") transport.receive("don", "too vague, name the files") replan_context = [e.event for e in client.calls[1]["events"]] assert any( isinstance(e, UserMessage) and "too vague" in e.text for e in replan_context ) assert "v2" in transport.outbound[-1].text def test_budget_exhaustion_is_not_retried(store: StateStore) -> None: """A budget will not reappear within a turn, so a retry buys nothing and delays telling the operator.""" class BrokeClient: def __init__(self) -> None: self.calls = 0 def complete(self, *args: Any, **kwargs: Any): self.calls += 1 raise BudgetExhausted("daily budget exhausted") client = BrokeClient() with pytest.raises(BudgetExhausted): LLMPlanningAgent(client, ROLES).plan(log(store), "arch", "demo") assert client.calls == 1 # not two