dongarandClaude Opus 5
bacf65dbe2
Found while chasing a flaky live test, but the flakiness was a real bug: only the ask_human path recorded a pending item, so AWAIT_INPUT reached any other way — planning failure, stopped milestone, boot reconciliation — refused every operator reply with "no pending question to answer" and the agent could never be recovered. The operator had no way back short of wiping state. Every path into AWAIT_INPUT now records what a reply should resume, and answering dispatches on it: planning failures and questions resume planning, a stopped milestone returns to its still-approved gate, and the fallback says what commands work instead of dead-ending. Two regression tests cover the recovery paths. The live test also scripted a fixed approve/approve/start sequence, which a real model breaks by legitimately calling ask_human at a gate. It now plays the operator adaptively, responding to whatever the agent actually did. Five consecutive live runs pass. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
108 lines
3.8 KiB
Python
108 lines
3.8 KiB
Python
"""End-to-end milestone against real models: plan with the planner role,
|
|
approve through the gate, then let the executor role actually create and test
|
|
a file on disk.
|
|
|
|
Gated on NISSE_LIVE_PLANNER_MODEL like the planning test — it needs a model
|
|
that holds the tool-call contract on both ends.
|
|
"""
|
|
|
|
import io
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from nisse_dev.control import (
|
|
AWAIT_ARCH_SIGNOFF,
|
|
AWAIT_INPUT,
|
|
AWAIT_MILESTONE_SIGNOFF,
|
|
EXECUTING,
|
|
Orchestrator,
|
|
)
|
|
from nisse_dev.envfile import load_env_file
|
|
from nisse_dev.executor import LLMExecutor
|
|
from nisse_dev.llm import LiteLLMClient, RoleMap
|
|
from nisse_dev.oplog import OpLog
|
|
from nisse_dev.planner import LLMPlanningAgent
|
|
from nisse_dev.state import StateStore
|
|
from nisse_dev.tools import Workspace
|
|
from nisse_dev.transport import InMemoryTransport
|
|
|
|
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
|
|
|
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
|
|
|
pytestmark = [
|
|
pytest.mark.integration,
|
|
pytest.mark.skipif(
|
|
not os.environ.get(SENTINEL),
|
|
reason=f"no live planner route configured ({SENTINEL} unset)",
|
|
),
|
|
]
|
|
|
|
|
|
def _drive_to_execution(
|
|
store: StateStore, transport: InMemoryTransport, max_steps: int = 10
|
|
) -> None:
|
|
"""Play the operator. A scripted approve/approve/start sequence is not
|
|
robust against a live model: ask_human is a legitimate move at any gate,
|
|
and a question shifts the state machine under a fixed script. So respond
|
|
to whatever the agent actually did."""
|
|
for _ in range(max_steps):
|
|
state = store.get_state()
|
|
assert state is not None
|
|
if state.state == EXECUTING:
|
|
return
|
|
gate = state.data.get("gate") or {}
|
|
if state.state == AWAIT_INPUT:
|
|
transport.receive("don", "Use your best judgement and proceed.")
|
|
elif state.state == AWAIT_MILESTONE_SIGNOFF and gate.get("approved"):
|
|
transport.receive("don", "start milestone 1")
|
|
elif state.state in (AWAIT_ARCH_SIGNOFF, AWAIT_MILESTONE_SIGNOFF):
|
|
transport.receive("don", "approve")
|
|
else:
|
|
raise AssertionError(f"unexpected state {state.state}")
|
|
raise AssertionError(
|
|
f"never reached execution; last message: {transport.outbound[-1]}"
|
|
)
|
|
|
|
|
|
def test_plan_approve_execute_produces_a_real_file(tmp_path: Path) -> None:
|
|
assert os.environ.get("NISSE_LLM_TOKEN"), (
|
|
f"{SENTINEL} is set, so the live path is expected to work, "
|
|
"but NISSE_LLM_TOKEN is missing"
|
|
)
|
|
# Both roles use the capable model here: the point is the whole gated
|
|
# loop over a real proxy, not local-model competence (measured separately).
|
|
model = os.environ[SENTINEL]
|
|
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
|
client = LiteLLMClient.from_env()
|
|
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
|
workspace = Workspace(tmp_path / "work", timeout_s=60)
|
|
transport = InMemoryTransport()
|
|
orchestrator = Orchestrator(
|
|
store,
|
|
transport,
|
|
LLMPlanningAgent(client, store, roles),
|
|
operators={"don"},
|
|
roles=roles,
|
|
executor=LLMExecutor(client, store, roles, workspace, max_turns=10),
|
|
)
|
|
orchestrator.start()
|
|
|
|
transport.receive(
|
|
"don",
|
|
"use greet. Milestone 1: create greet.py in the workspace root defining "
|
|
"greet(name) returning 'Hello, <name>!', and verify it by running python.",
|
|
)
|
|
_drive_to_execution(store, transport)
|
|
|
|
state = store.get_state()
|
|
assert state is not None
|
|
assert state.state in (EXECUTING, AWAIT_INPUT)
|
|
|
|
created = list(workspace.root.rglob("*.py"))
|
|
assert created, f"executor created no Python file; said: {transport.outbound[-1]}"
|
|
combined = "\n".join(p.read_text() for p in created)
|
|
assert "def greet" in combined
|