M2.8: switch over to the new loop
nisse-dev now runs only the conversational loop. The gated state machine it was built alongside is gone: - Removed control.py, planner.py, executor.py, approvals.py, gates.py (command_form moves to loop/reader.py), publish.py and poller.py, the old assembly in agent.py (its shared helpers move to loop/build.py), the store's state-machine view (get_state, transition), their tests, the old end-to-end and live planning/milestone/gate tests, and specs/orchestrator.qnt with its scenarios. - Roles are driver, helper, expert and classifier; driver is required. loop:, roles.planner, roles.executor and tuning.executor are refused with what to use instead; tuning.driver takes the turn cap and failure limit. - The benchmark and the capability matrix measure the driver: the loop's own system prompt and tools, the task's plan as the agreed restatement. Task plans end with finish_milestone, so earlier runs show as (old); stored runs' executor tuning reads as the driver's. - A delegation on the loop records what its helper's alias resolved to (L9), as the old path's did. - The dump shows a work item's revision and what it waits for. - Behaviour the old tests covered and the loop shares stays covered: driver runs that fail on a model call, stop mid-turn, reset the failure count; go-aheads ignoring case and punctuation; a request too big to fit is never sent; the assembly tests, now on build_loop_agent. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
1 parent
09232dcbdd
commit
6ae2375194
89 files changed
+951
-8592
No files matched your search
@@ -30,11 +30,10 @@ from nisse_dev.canonical import (
|
||||
)
|
||||
from nisse_dev.catalog import ModelCatalog, select_client
|
||||
from nisse_dev.config import AgentConfig
|
||||
from nisse_dev.executor import FINISH_TOOL
|
||||
from nisse_dev.expert import parse_answer
|
||||
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError
|
||||
from nisse_dev.planner import PLANNER_TOOLS
|
||||
from nisse_dev.tools import EXECUTOR_TOOLS
|
||||
from nisse_dev.loop.tools import ASK_HUMAN, FINISH_MILESTONE, RESTATE
|
||||
from nisse_dev.tools import EXECUTION_TOOLS
|
||||
|
||||
pytestmark = pytest.mark.integration
|
||||
|
||||
@@ -99,44 +98,42 @@ def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None
|
||||
assert texts, f"{model} produced no text: {produced!r}"
|
||||
|
||||
|
||||
def test_model_holds_the_planner_contract(
|
||||
def test_model_holds_the_restatement_contract(
|
||||
model: str, catalog: ModelCatalog, routes
|
||||
) -> None:
|
||||
"""L3's approvals are structural at the tool layer: a model that answers in
|
||||
prose cannot be gated, because there is no plan to approve."""
|
||||
"""L3's go-ahead is structural at the tool layer: a driver that answers
|
||||
in prose cannot be gated, because there is no restatement to agree to."""
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
produced = client.complete(
|
||||
user("Draft an architecture plan for a command-line slug generator."),
|
||||
PLANNER_TOOLS,
|
||||
user("Issue #1: add a command-line slug generator."),
|
||||
[RESTATE, ASK_HUMAN],
|
||||
model,
|
||||
system=(
|
||||
"You must answer by calling exactly one tool: submit_plan or "
|
||||
"You must answer by calling exactly one tool: restate or "
|
||||
"ask_human. Plain prose is not accepted."
|
||||
),
|
||||
)
|
||||
assert_answered(client, model, catalog)
|
||||
calls = [e for e in produced if isinstance(e, ToolCall)]
|
||||
assert calls, f"{model} answered in prose instead of calling a tool"
|
||||
assert calls[0].tool in {"submit_plan", "ask_human"}
|
||||
if calls[0].tool == "submit_plan":
|
||||
content = calls[0].arguments.get("content", "")
|
||||
assert calls[0].tool in {"restate", "ask_human"}
|
||||
if calls[0].tool == "restate":
|
||||
content = calls[0].arguments.get("text", "")
|
||||
# llama3.2 emitted "{" here — structurally a tool call, substantively
|
||||
# nothing. Length is a crude proxy, but it separates the two.
|
||||
assert len(str(content)) > 80, (
|
||||
f"{model} submitted a degenerate plan: {content!r}"
|
||||
)
|
||||
assert len(str(content)) > 80, f"{model} restated degenerately: {content!r}"
|
||||
|
||||
|
||||
def test_model_round_trips_a_tool_result(
|
||||
model: str, catalog: ModelCatalog, routes
|
||||
) -> None:
|
||||
"""The executor loop feeds results back; a model that cannot consume its
|
||||
"""The driver's run feeds results back; a model that cannot consume its
|
||||
own tool result cannot run a milestone."""
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
events = user("List the files in the current directory using your tools.")
|
||||
produced = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.")
|
||||
produced = client.complete(events, EXECUTION_TOOLS, model, system="Use a tool.")
|
||||
calls = [e for e in produced if isinstance(e, ToolCall)]
|
||||
assert calls, f"{model} made no tool call"
|
||||
|
||||
@@ -154,7 +151,7 @@ def test_model_round_trips_a_tool_result(
|
||||
)
|
||||
)
|
||||
assert_answered(client, model, catalog)
|
||||
follow_up = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.")
|
||||
follow_up = client.complete(events, EXECUTION_TOOLS, model, system="Use a tool.")
|
||||
assert follow_up, f"{model} produced nothing after its tool result"
|
||||
|
||||
|
||||
@@ -203,7 +200,7 @@ def test_route_selection_matches_the_model(
|
||||
assert_answered(chosen, model, catalog)
|
||||
|
||||
|
||||
FINISH_TOOLS = [*EXECUTOR_TOOLS, FINISH_TOOL]
|
||||
FINISH_TOOLS = [*EXECUTION_TOOLS, FINISH_MILESTONE]
|
||||
OFFERED = {tool["function"]["name"] for tool in FINISH_TOOLS}
|
||||
|
||||
|
||||
@@ -212,7 +209,7 @@ def test_model_calls_only_the_tools_it_was_offered(
|
||||
) -> None:
|
||||
"""Measured on a server that did not enforce tool_choice: a model asked
|
||||
to chat called "chat", "say_hello" and "respond", none of them offered.
|
||||
The executor counts those as failures; this measures how often."""
|
||||
The driver answers those as failures; this measures how often."""
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
produced = client.complete(
|
||||
|
||||
Reference in new issue
Block a user