Merge pull request 'Add executor tool loop (A12/A15) — step 5b' (#12) from executor-loop into main

tests/integration/test_milestone_live.py

@@ -0,0 +1,107 @@
+"""End-to-end milestone against real models: plan with the planner role,
+approve through the gate, then let the executor role actually create and test
+a file on disk.
+
+Gated on NISSE_LIVE_PLANNER_MODEL like the planning test — it needs a model
+that holds the tool-call contract on both ends.
+"""
+
+import io
+import os
+from pathlib import Path
+
+import pytest
+
+from nisse_dev.control import (
+    AWAIT_ARCH_SIGNOFF,
+    AWAIT_INPUT,
+    AWAIT_MILESTONE_SIGNOFF,
+    EXECUTING,
+    Orchestrator,
+)
+from nisse_dev.envfile import load_env_file
+from nisse_dev.executor import LLMExecutor
+from nisse_dev.llm import LiteLLMClient, RoleMap
+from nisse_dev.oplog import OpLog
+from nisse_dev.planner import LLMPlanningAgent
+from nisse_dev.state import StateStore
+from nisse_dev.tools import Workspace
+from nisse_dev.transport import InMemoryTransport
+
+load_env_file(Path(__file__).resolve().parents[2] / ".env")
+
+SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
+
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.skipif(
+        not os.environ.get(SENTINEL),
+        reason=f"no live planner route configured ({SENTINEL} unset)",
+    ),
+]
+
+
+def _drive_to_execution(
+    store: StateStore, transport: InMemoryTransport, max_steps: int = 10
+) -> None:
+    """Play the operator. A scripted approve/approve/start sequence is not
+    robust against a live model: ask_human is a legitimate move at any gate,
+    and a question shifts the state machine under a fixed script. So respond
+    to whatever the agent actually did."""
+    for _ in range(max_steps):
+        state = store.get_state()
+        assert state is not None
+        if state.state == EXECUTING:
+            return
+        gate = state.data.get("gate") or {}
+        if state.state == AWAIT_INPUT:
+            transport.receive("don", "Use your best judgement and proceed.")
+        elif state.state == AWAIT_MILESTONE_SIGNOFF and gate.get("approved"):
+            transport.receive("don", "start milestone 1")
+        elif state.state in (AWAIT_ARCH_SIGNOFF, AWAIT_MILESTONE_SIGNOFF):
+            transport.receive("don", "approve")
+        else:
+            raise AssertionError(f"unexpected state {state.state}")
+    raise AssertionError(
+        f"never reached execution; last message: {transport.outbound[-1]}"
+    )
+
+
+def test_plan_approve_execute_produces_a_real_file(tmp_path: Path) -> None:
+    assert os.environ.get("NISSE_LLM_TOKEN"), (
+        f"{SENTINEL} is set, so the live path is expected to work, "
+        "but NISSE_LLM_TOKEN is missing"
+    )
+    # Both roles use the capable model here: the point is the whole gated
+    # loop over a real proxy, not local-model competence (measured separately).
+    model = os.environ[SENTINEL]
+    roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
+    client = LiteLLMClient.from_env()
+    store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
+    workspace = Workspace(tmp_path / "work", timeout_s=60)
+    transport = InMemoryTransport()
+    orchestrator = Orchestrator(
+        store,
+        transport,
+        LLMPlanningAgent(client, store, roles),
+        operators={"don"},
+        roles=roles,
+        executor=LLMExecutor(client, store, roles, workspace, max_turns=10),
+    )
+    orchestrator.start()
+
+    transport.receive(
+        "don",
+        "use greet. Milestone 1: create greet.py in the workspace root defining "
+        "greet(name) returning 'Hello, <name>!', and verify it by running python.",
+    )
+    _drive_to_execution(store, transport)
+
+    state = store.get_state()
+    assert state is not None
+    assert state.state in (EXECUTING, AWAIT_INPUT)
+
+    created = list(workspace.root.rglob("*.py"))
+    assert created, f"executor created no Python file; said: {transport.outbound[-1]}"
+    combined = "\n".join(p.read_text() for p in created)
+    assert "def greet" in combined