"""End-to-end milestone against real models: plan with the planner role, approve through the gate, then let the executor role actually create and test a file on disk. Uses the test config's planner model for every role: it needs a model that holds the tool-call contract on both ends. """ import io from pathlib import Path import pytest from nisse_dev.control import ( AWAIT_ARCH_SIGNOFF, AWAIT_INPUT, AWAIT_MILESTONE_SIGNOFF, EXECUTING, PR_OPEN, Orchestrator, ) from nisse_dev.executor import LLMExecutor from nisse_dev.llm import LiteLLMClient, RoleMap from nisse_dev.oplog import OpLog from nisse_dev.planner import LLMPlanningAgent from nisse_dev.state import StateStore from nisse_dev.tools import Workspace from nisse_dev.transport import InMemoryTransport pytestmark = pytest.mark.integration def _drive_to_execution( store: StateStore, transport: InMemoryTransport, max_steps: int = 10 ) -> None: """Play the operator. A scripted approve/approve/start sequence is not robust against a live model: ask_human is a legitimate move at any gate, and a question shifts the state machine under a fixed script. So respond to whatever the agent actually did.""" for _ in range(max_steps): state = store.get_state() assert state is not None if state.state == EXECUTING: return gate = state.data.get("gate") or {} if state.state == AWAIT_INPUT: transport.receive("don", "Use your best judgement and proceed.") elif state.state == AWAIT_MILESTONE_SIGNOFF and gate.get("approved"): # The milestone runs to completion inside this command. transport.receive("don", "start milestone 1") return elif state.state in (AWAIT_ARCH_SIGNOFF, AWAIT_MILESTONE_SIGNOFF): transport.receive("don", "approve") else: raise AssertionError(f"unexpected state {state.state}") raise AssertionError( f"never reached execution; last message: {transport.outbound[-1]}" ) def test_plan_approve_execute_produces_a_real_file( tmp_path: Path, litellm: LiteLLMClient, live_model: str ) -> None: # Both roles use the capable model here: the point is the whole gated # loop over a real proxy, not local-model competence (measured separately). model = live_model roles = RoleMap(executor=model, planner=model, helper=model, expert=model) client = litellm store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO())) workspace = Workspace(tmp_path / "work", timeout_s=60) transport = InMemoryTransport() orchestrator = Orchestrator( store, transport, LLMPlanningAgent(client, roles), operators={"don"}, roles=roles, executor=LLMExecutor(client, roles, workspace, max_turns=10), ) orchestrator.start() transport.receive( "don", "use greet. Milestone 1: create greet.py in the workspace root defining " "greet(name) returning 'Hello, !', and verify it by running python.", ) _drive_to_execution(store, transport) state = store.get_state() assert state is not None # Never EXECUTING at rest: with no publisher a finished milestone waits in # PR_OPEN, and one that stopped short waits for the operator. assert state.state in (PR_OPEN, AWAIT_INPUT) created = list(workspace.root.rglob("*.py")) assert created, f"executor created no Python file; said: {transport.outbound[-1]}" combined = "\n".join(p.read_text() for p in created) assert "def greet" in combined