dongarandClaude Opus 5
cf85648719
- planner.py: LLMPlanningAgent offers exactly two tools, submit_plan and ask_human, so gating stays structural at the tool layer; no proceed tool exists. System prompt requires explicit, low-inference plans because a weaker model executes them (A13) - Model output is appended to the canonical log as it arrives, before being interpreted, so a crash mid-planning leaves a truthful transcript - A bad turn (prose instead of a tool call, malformed arguments, or a failed call) retries once, then raises PlanningError; the orchestrator gates to AWAIT_INPUT rather than inventing a plan - Orchestrator: appends every operator message, plan_submitted, and gate_opened to the transcript, so re-planning sees rejection feedback as conversation rather than out-of-band parameters; pins the A11 role mapping at task start - llm.py: non-JSON or non-object tool arguments now raise LLMError instead of escaping as JSONDecodeError — observed from llama3.2, which emits truncated argument strings - Live planning test is opt-in on NISSE_LIVE_PLANNER_MODEL: planning escalates unconditionally (A12) and the local model does not hold the tool-call contract, so it awaits a frontier route Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
65 lines
2.3 KiB
Python
65 lines
2.3 KiB
Python
"""Live planning turn: a real model must answer with a submit_plan or
|
|
ask_human tool call, not prose.
|
|
|
|
Deliberately opt-in via NISSE_LIVE_PLANNER_MODEL rather than the usual
|
|
NISSE_LLM_BASE_URL sentinel. Planning escalates to a frontier model
|
|
unconditionally (A12), and the small local model does not hold the tool-call
|
|
contract reliably — measured, see the PR — so this test needs a frontier
|
|
route that does not exist yet. Set the variable to the model to plan with
|
|
once one is configured; until then this skips on purpose, not by accident.
|
|
"""
|
|
|
|
import io
|
|
import os
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
from nisse_dev.canonical import UserMessage
|
|
from nisse_dev.control import AskHuman, SubmitPlan
|
|
from nisse_dev.envfile import load_env_file
|
|
from nisse_dev.llm import LiteLLMClient, RoleMap
|
|
from nisse_dev.oplog import OpLog
|
|
from nisse_dev.planner import LLMPlanningAgent
|
|
from nisse_dev.state import StateStore
|
|
|
|
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
|
|
|
SENTINEL = "NISSE_LIVE_PLANNER_MODEL"
|
|
|
|
pytestmark = [
|
|
pytest.mark.integration,
|
|
pytest.mark.skipif(
|
|
not os.environ.get(SENTINEL),
|
|
reason=f"no frontier planner route configured ({SENTINEL} unset)",
|
|
),
|
|
]
|
|
|
|
|
|
def test_live_planning_turn_returns_a_tool_action(tmp_path: Path) -> None:
|
|
assert os.environ.get("NISSE_LLM_TOKEN"), (
|
|
f"{SENTINEL} is set, so the live planning path is expected to work, "
|
|
"but NISSE_LLM_TOKEN is missing"
|
|
)
|
|
model = os.environ[SENTINEL]
|
|
roles = RoleMap(executor=model, planner=model, delegate=model, consultant=model)
|
|
store = StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
|
|
store.append_event(
|
|
UserMessage(
|
|
text=(
|
|
"Project 'hello-cli': a Python script that prints a greeting. "
|
|
"Draft the architecture plan."
|
|
),
|
|
source="live-test",
|
|
)
|
|
)
|
|
agent = LLMPlanningAgent(LiteLLMClient.from_env(), store, roles)
|
|
|
|
action = agent.plan("arch", "hello-cli")
|
|
|
|
assert isinstance(action, SubmitPlan | AskHuman)
|
|
text = action.content if isinstance(action, SubmitPlan) else action.question
|
|
assert text.strip(), "tool call carried empty content"
|
|
# The turn must be durable regardless of what the model said (A2/A10).
|
|
assert len(store.read_events()) > 1
|