dongarandClaude Opus 5.5
b934c46714
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 56s
The executor ended a milestone on any reply without a tool call, and
nothing held a model to the offered tools. Measured on the 4070:
tool_choice "required" is enforced by vLLM (local-fast) and Anthropic,
but qwen3-14b (local-coder) under it cannot stop and writes until the
output ceiling, a known model bug. So enforcement is asked for per
model, and the loop holds the contract itself everywhere.
- The catalog reads supports_tool_choice from LiteLLM's model_info;
BudgetedClient sends tool_choice only to models declared true, as
"required" on the normalized route and {"type": "any"} on the
Anthropic passthrough, and only when tools are offered
- The executor gains an explicit finish tool, the only way to end a
milestone. Finishing with nothing changed is refused once; a prose
reply gets one nudge, then escalates; a call to a tool not offered
counts toward failure_limit, like a failed command; calls after
finish are recorded as not run
- Clients record the model that actually answered. ModelFacts.answered
recognises a proxy fallback, which once credited Claude's answers to
a local model that could not call tools at all
- The context probe reports a fallback instead of trusting it; rerun,
both local models now refuse over-long prompts, as they should
- The capability matrix checks every answer came from the model asked,
that models call only offered tools, and that a declared
supports_tool_choice is true
- Spec A15 and the tuning guide say how enforcement is requested,
declared and backed up
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
350 lines
12 KiB
Python
350 lines
12 KiB
Python
"""The composition root. These tests exist because two components — the
|
|
delegation harness and the comment poller — were fully built, fully tested,
|
|
and reachable from nothing but their own tests. Unit tests that construct a
|
|
component directly cannot catch that; only assembly can."""
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
from test_config import SECRETS, tree
|
|
|
|
from nisse_dev.agent import build_agent, discover_models, prepare_workspace
|
|
from nisse_dev.catalog import ModelCatalog, UnknownModel
|
|
from nisse_dev.config import AgentConfig
|
|
from nisse_dev.git import GitRepo
|
|
from nisse_dev.oplog import OpLog
|
|
|
|
# What the test roles resolve to, as LiteLLM would report them.
|
|
TEST_CATALOG = ModelCatalog.from_payload(
|
|
{
|
|
"data": [
|
|
{
|
|
"model_name": alias,
|
|
"litellm_params": {"model": f"anthropic/{alias}"},
|
|
"model_info": {
|
|
"litellm_provider": "anthropic",
|
|
"supports_function_calling": True,
|
|
"max_input_tokens": 200000,
|
|
"max_output_tokens": 64000,
|
|
},
|
|
}
|
|
for alias in ("claude-opus-5", "claude-haiku-4-5")
|
|
]
|
|
}
|
|
)
|
|
|
|
|
|
def config_for(tmp_path: Path, **changes: str) -> AgentConfig:
|
|
return AgentConfig.from_tree(
|
|
tree(
|
|
state_db=str(tmp_path / "state.db"),
|
|
workspace=str(tmp_path / "work"),
|
|
**changes,
|
|
),
|
|
SECRETS,
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def origin(tmp_path: Path) -> Path:
|
|
"""A bare repo standing in for the project on the forge."""
|
|
remote = tmp_path / "origin.git"
|
|
GitRepo(tmp_path)._run("init", "-q", "--bare", "-b", "main", str(remote))
|
|
seed = GitRepo(tmp_path / "seed")
|
|
seed.init()
|
|
seed.configure_identity("seed", "seed@test")
|
|
(seed.root / "README.md").write_text("project\n")
|
|
seed.commit_all("seed")
|
|
seed.push(str(remote), "main")
|
|
return remote
|
|
|
|
|
|
def test_workspace_is_cloned_when_absent(tmp_path: Path, origin: Path) -> None:
|
|
config = config_for(tmp_path, gitea__url=f"file://{origin}")
|
|
# authenticated_url would mangle a file:// remote, so clone directly.
|
|
GitRepo(tmp_path).clone(str(origin), config.workspace)
|
|
repo = prepare_workspace(config, "nisse-dev-testing")
|
|
|
|
assert (config.workspace / "README.md").exists()
|
|
assert repo.current_branch() == "main"
|
|
|
|
|
|
def test_workspace_is_reused_when_already_cloned(tmp_path: Path, origin: Path) -> None:
|
|
config = config_for(tmp_path)
|
|
GitRepo(tmp_path).clone(str(origin), config.workspace)
|
|
(config.workspace / "local-work.txt").write_text("in progress")
|
|
|
|
prepare_workspace(config, "nisse-dev-testing")
|
|
|
|
# Re-cloning would silently destroy uncommitted work.
|
|
assert (config.workspace / "local-work.txt").exists()
|
|
|
|
|
|
class FakeTransport:
|
|
def __init__(self) -> None:
|
|
self.sent: list[str] = []
|
|
self.started = False
|
|
self.closed = False
|
|
|
|
def send_message(self, text: str) -> None:
|
|
self.sent.append(text)
|
|
|
|
def send_approval_request(self, text: str) -> None:
|
|
self.sent.append(text)
|
|
|
|
def ask_question(self, text: str) -> None:
|
|
self.sent.append(text)
|
|
|
|
def on_message(self, callback: Any) -> None:
|
|
self.callback = callback
|
|
|
|
def start(self) -> None:
|
|
self.started = True
|
|
|
|
def close(self) -> None:
|
|
self.closed = True
|
|
|
|
|
|
def build_offline(tmp_path: Path, origin: Path, monkeypatch: Any) -> Any:
|
|
"""Assemble the real graph with only the network boundaries faked."""
|
|
config = config_for(tmp_path)
|
|
GitRepo(tmp_path).clone(str(origin), config.workspace)
|
|
|
|
monkeypatch.setattr(
|
|
"nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"
|
|
)
|
|
monkeypatch.setattr(
|
|
"nisse_dev.agent.ModelCatalog.discover",
|
|
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
|
)
|
|
monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport())
|
|
monkeypatch.setattr(
|
|
"nisse_dev.agent.authenticated_url",
|
|
lambda base, repo, user, token: str(origin),
|
|
)
|
|
return build_agent(config)
|
|
|
|
|
|
def test_the_whole_graph_assembles(tmp_path: Path, origin: Path, monkeypatch) -> None:
|
|
agent = build_offline(tmp_path, origin, monkeypatch)
|
|
orchestrator = agent.orchestrator
|
|
|
|
# Every component that was previously unreachable is now connected.
|
|
assert orchestrator._executor is not None
|
|
assert orchestrator._publisher is not None
|
|
assert orchestrator._board is not None
|
|
assert orchestrator._delegate is not None
|
|
assert orchestrator._poller_factory is not None
|
|
assert orchestrator._executor._delegate is not None # A12 escalation path
|
|
assert orchestrator._executor._expert is not None # A14 consult path
|
|
agent.stop()
|
|
|
|
|
|
def test_start_announces_where_it_resumed(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
agent = build_offline(tmp_path, origin, monkeypatch)
|
|
agent.start()
|
|
|
|
assert agent.transport.started
|
|
readiness = agent.transport.sent[-1]
|
|
assert "nisse-dev up as nisse-dev-testing" in readiness
|
|
assert "greet" in readiness # the project
|
|
assert "claude-opus-5" in readiness # the pinned planner
|
|
agent.stop()
|
|
|
|
|
|
def test_stop_closes_everything_and_releases_the_wait(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
agent = build_offline(tmp_path, origin, monkeypatch)
|
|
agent.start()
|
|
agent.stop()
|
|
|
|
assert agent.transport.closed
|
|
agent.wait() # returns immediately rather than hanging
|
|
assert agent.store.closed
|
|
|
|
|
|
def test_poller_factory_ignores_the_agents_own_comments(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
# Without this the agent answers its own PR replies in a loop.
|
|
agent = build_offline(tmp_path, origin, monkeypatch)
|
|
poller = agent.orchestrator._poller_factory(lambda c: None)
|
|
assert "nisse-dev-testing" in poller._ignore
|
|
agent.stop()
|
|
|
|
|
|
def test_delegation_uses_the_passthrough_route(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
# A13: delegations run vendor-native; the normalized route is for the
|
|
# places where fidelity is cheap.
|
|
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
|
|
|
|
agent = build_offline(tmp_path, origin, monkeypatch)
|
|
delegation = agent.orchestrator._delegate
|
|
assert isinstance(delegation._client._inner, AnthropicPassthroughClient)
|
|
assert isinstance(agent.orchestrator._executor._client._inner, LiteLLMClient)
|
|
agent.stop()
|
|
|
|
|
|
def test_httpx_clients_are_not_shared_between_routes(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
agent = build_offline(tmp_path, origin, monkeypatch)
|
|
normalized = agent.orchestrator._executor._client._inner._http
|
|
passthrough = agent.orchestrator._delegate._client._inner._http
|
|
assert isinstance(normalized, httpx.Client)
|
|
assert normalized is not passthrough # different base URLs and headers
|
|
agent.stop()
|
|
|
|
|
|
# --- startup model discovery (A5/A11) ------------------------------------
|
|
|
|
|
|
def test_a_typod_role_refuses_to_boot(tmp_path: Path, monkeypatch) -> None:
|
|
# Misconfiguration should cost a second at startup, not be discovered ten
|
|
# minutes into a milestone.
|
|
config = config_for(tmp_path, roles__planner="clause-opus-5")
|
|
monkeypatch.setattr(
|
|
"nisse_dev.agent.ModelCatalog.discover",
|
|
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
|
)
|
|
with pytest.raises(UnknownModel, match="planner='clause-opus-5'"):
|
|
discover_models(config, OpLog("test"))
|
|
|
|
|
|
def test_a_role_this_key_cannot_reach_refuses_to_boot(
|
|
tmp_path: Path, monkeypatch
|
|
) -> None:
|
|
# /model/info is scoped per key, so the same check catches a role pointed
|
|
# at a real model that this token has no access to.
|
|
config = config_for(tmp_path, roles__executor="local-fast")
|
|
monkeypatch.setattr(
|
|
"nisse_dev.agent.ModelCatalog.discover",
|
|
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
|
)
|
|
with pytest.raises(UnknownModel, match="executor='local-fast'"):
|
|
discover_models(config, OpLog("test"))
|
|
|
|
|
|
def test_a_tool_less_model_warns_but_still_boots(tmp_path: Path, monkeypatch) -> None:
|
|
# A15 expects weak local models to be accommodated, and models LiteLLM
|
|
# declares tool-less are observed driving tool loops anyway.
|
|
import io
|
|
|
|
tool_less = ModelCatalog.from_payload(
|
|
{
|
|
"data": [
|
|
{
|
|
"model_name": alias,
|
|
"litellm_params": {"model": "openai/qwen3-8b-awq"},
|
|
"model_info": {
|
|
"litellm_provider": "openai",
|
|
"supports_function_calling": False,
|
|
"max_input_tokens": 32768,
|
|
},
|
|
}
|
|
for alias in ("claude-opus-5", "claude-haiku-4-5")
|
|
]
|
|
}
|
|
)
|
|
monkeypatch.setattr(
|
|
"nisse_dev.agent.ModelCatalog.discover",
|
|
classmethod(lambda cls, *a, **kw: tool_less),
|
|
)
|
|
stream = io.StringIO()
|
|
discover_models(config_for(tmp_path), OpLog("test", stream))
|
|
|
|
logged = stream.getvalue()
|
|
assert "model_tools_unsupported" in logged
|
|
assert '"role": "executor"' in logged
|
|
assert '"role": "consultant"' not in logged # ask_expert drives no tools
|
|
|
|
|
|
def test_the_resolved_model_is_pinned_at_task_start(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
agent = build_offline(tmp_path, origin, monkeypatch)
|
|
# The store belongs to the worker thread, so the work has to run there.
|
|
agent.worker.call(lambda: agent.orchestrator._begin_project("greet"))
|
|
|
|
pinned = agent.orchestrator._data["models"]
|
|
assert pinned["planner"] == {
|
|
"alias": "claude-opus-5",
|
|
"backing": "anthropic/claude-opus-5",
|
|
"provider": "anthropic",
|
|
}
|
|
agent.stop()
|
|
|
|
|
|
def test_delegation_records_what_the_alias_resolved_to(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
# A16 eval history keyed on an alias alone would average across whatever
|
|
# that alias pointed at over time.
|
|
agent = build_offline(tmp_path, origin, monkeypatch)
|
|
assert agent.orchestrator._delegate._backing_model == "anthropic/claude-opus-5"
|
|
agent.stop()
|
|
|
|
|
|
def test_tuning_reaches_every_component(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
from dataclasses import replace
|
|
|
|
from nisse_dev.config import (
|
|
DelegationTuning,
|
|
ExecutorTuning,
|
|
ExpertTuning,
|
|
ToolsTuning,
|
|
Tuning,
|
|
)
|
|
|
|
config = replace(
|
|
config_for(tmp_path),
|
|
max_output_tokens=8192,
|
|
tuning=Tuning(
|
|
executor=ExecutorTuning(max_turns=40, failure_limit=3),
|
|
expert=ExpertTuning(consult_budget=5),
|
|
delegation=DelegationTuning(
|
|
max_turns=12, max_tokens=50_000, wall_clock_s=60
|
|
),
|
|
tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000),
|
|
),
|
|
)
|
|
GitRepo(tmp_path).clone(str(origin), config.workspace)
|
|
monkeypatch.setattr(
|
|
"nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"
|
|
)
|
|
monkeypatch.setattr(
|
|
"nisse_dev.agent.ModelCatalog.discover",
|
|
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
|
)
|
|
monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport())
|
|
monkeypatch.setattr(
|
|
"nisse_dev.agent.authenticated_url",
|
|
lambda base, repo, user, token: str(origin),
|
|
)
|
|
agent = build_agent(config)
|
|
executor = agent.orchestrator._executor
|
|
delegation = agent.orchestrator._delegate
|
|
|
|
assert (executor._max_turns, executor._failure_limit) == (40, 3)
|
|
assert executor._expert._budget == 5
|
|
assert delegation._caps.turns == 12
|
|
assert delegation._caps.tokens == 50_000
|
|
assert delegation._caps.wall_clock_s == 60
|
|
assert executor._workspace._timeout_s == 300
|
|
assert executor._workspace._max_output_chars == 2000
|
|
assert executor._client._inner._max_tokens == 8192
|
|
assert delegation._client._inner._max_tokens == 8192
|
|
assert executor._client._max_output_tokens == 8192
|
|
# tool_choice is asked of exactly the models LiteLLM declares enforce it.
|
|
assert executor._client._enforces_tool_choice("claude-opus-5") is False
|
|
agent.stop()
|