dongarandClaude Opus 5.5
6ae2375194
nisse-dev now runs only the conversational loop. The gated state machine it was built alongside is gone: - Removed control.py, planner.py, executor.py, approvals.py, gates.py (command_form moves to loop/reader.py), publish.py and poller.py, the old assembly in agent.py (its shared helpers move to loop/build.py), the store's state-machine view (get_state, transition), their tests, the old end-to-end and live planning/milestone/gate tests, and specs/orchestrator.qnt with its scenarios. - Roles are driver, helper, expert and classifier; driver is required. loop:, roles.planner, roles.executor and tuning.executor are refused with what to use instead; tuning.driver takes the turn cap and failure limit. - The benchmark and the capability matrix measure the driver: the loop's own system prompt and tools, the task's plan as the agreed restatement. Task plans end with finish_milestone, so earlier runs show as (old); stored runs' executor tuning reads as the driver's. - A delegation on the loop records what its helper's alias resolved to (L9), as the old path's did. - The dump shows a work item's revision and what it waits for. - Behaviour the old tests covered and the loop shares stays covered: driver runs that fail on a model call, stop mid-turn, reset the failure count; go-aheads ignoring case and punctuation; a request too big to fit is never sent; the assembly tests, now on build_loop_agent. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
405 lines
14 KiB
Python
405 lines
14 KiB
Python
"""The composition root. These tests exist because two components — the
|
|
delegation harness and the comment poller — were fully built, fully tested,
|
|
and reachable from nothing but their own tests. Unit tests that construct a
|
|
component directly cannot catch that; only assembly can."""
|
|
|
|
import subprocess
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import httpx
|
|
import pytest
|
|
from test_config import SECRETS, P, tree
|
|
|
|
from nisse_dev.catalog import ModelCatalog, UnknownModel
|
|
from nisse_dev.config import AgentConfig
|
|
from nisse_dev.execution import SSHHost
|
|
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
|
|
from nisse_dev.loop.build import RunningLoop, build_loop_agent, discover_models
|
|
from nisse_dev.loop.build import prepare_workspace as prepare
|
|
from nisse_dev.oplog import OpLog
|
|
|
|
# What the test roles resolve to, as LiteLLM would report them.
|
|
TEST_CATALOG = ModelCatalog.from_payload(
|
|
{
|
|
"data": [
|
|
{
|
|
"model_name": alias,
|
|
"litellm_params": {"model": f"anthropic/{alias}"},
|
|
"model_info": {
|
|
"litellm_provider": "anthropic",
|
|
"supports_function_calling": True,
|
|
"max_input_tokens": 200000,
|
|
"max_output_tokens": 64000,
|
|
},
|
|
}
|
|
for alias in ("claude-opus-5", "claude-haiku-4-5")
|
|
]
|
|
}
|
|
)
|
|
|
|
|
|
# The build VM for every test here: the session's throwaway sshd.
|
|
HOST: SSHHost
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _build_host(ssh_host: SSHHost) -> None:
|
|
global HOST
|
|
HOST = ssh_host
|
|
|
|
|
|
def config_for(tmp_path: Path, **changes: str) -> AgentConfig:
|
|
return AgentConfig.from_tree(
|
|
tree(
|
|
state_db=str(tmp_path / "state.db"),
|
|
**{
|
|
f"{P}workspace": str(tmp_path / "work"),
|
|
f"{P}build_host__address": HOST.address,
|
|
f"{P}build_host__user": HOST.user,
|
|
f"{P}build_host__port": str(HOST.port),
|
|
f"{P}build_host__key": str(HOST._key),
|
|
},
|
|
**changes,
|
|
),
|
|
SECRETS,
|
|
)
|
|
|
|
|
|
def git(*args: str) -> None:
|
|
subprocess.run(["git", *args], check=True, capture_output=True)
|
|
|
|
|
|
@pytest.fixture
|
|
def origin(tmp_path: Path) -> Path:
|
|
"""A bare repo standing in for the project on the forge."""
|
|
remote = tmp_path / "origin.git"
|
|
git("init", "-q", "--bare", "-b", "main", str(remote))
|
|
seed = tmp_path / "seed"
|
|
git("init", "-q", "-b", "main", str(seed))
|
|
(seed / "README.md").write_text("project\n")
|
|
git("-C", str(seed), "add", "-A")
|
|
git(
|
|
"-C",
|
|
str(seed),
|
|
"-c",
|
|
"user.name=s",
|
|
"-c",
|
|
"user.email=s@t",
|
|
"commit",
|
|
"-qm",
|
|
"s",
|
|
)
|
|
git("-C", str(seed), "push", "-q", str(remote), "HEAD:refs/heads/main")
|
|
return remote
|
|
|
|
|
|
def prepare_workspace(config: AgentConfig, origin: Path) -> Any:
|
|
[project] = config.projects
|
|
return prepare(HOST, config, "nisse-dev-testing", str(origin), None, project)
|
|
|
|
|
|
def test_workspace_is_cloned_on_the_build_host_when_absent(
|
|
tmp_path: Path, origin: Path
|
|
) -> None:
|
|
config = config_for(tmp_path)
|
|
repo = prepare_workspace(config, origin)
|
|
workspace = config.projects[0].workspace
|
|
assert (workspace / "README.md").exists()
|
|
assert repo.current_branch() == "main"
|
|
# The clone's configuration holds the plain URL it was given (#89).
|
|
assert str(origin) in (workspace / ".git" / "config").read_text()
|
|
|
|
|
|
def test_workspace_is_reused_when_already_cloned(tmp_path: Path, origin: Path) -> None:
|
|
config = config_for(tmp_path)
|
|
workspace = config.projects[0].workspace
|
|
prepare_workspace(config, origin)
|
|
(workspace / "local-work.txt").write_text("in progress")
|
|
|
|
prepare_workspace(config, origin)
|
|
|
|
# Re-cloning would silently destroy uncommitted work.
|
|
assert (workspace / "local-work.txt").exists()
|
|
|
|
|
|
class FakeTransport:
|
|
def __init__(self) -> None:
|
|
self.sent: list[str] = []
|
|
self.closed = False
|
|
|
|
def post(self, text: str, thread: str | None = None) -> str:
|
|
self.sent.append(text)
|
|
return f"$event{len(self.sent)}"
|
|
|
|
def on_message(self, callback: Any) -> None:
|
|
self.callback = callback
|
|
|
|
def close(self) -> None:
|
|
self.closed = True
|
|
|
|
|
|
def offline(monkeypatch: Any, origins: dict[str, str] | str, catalog=None) -> None:
|
|
"""Only the network boundaries faked: Gitea, LiteLLM's model list, Matrix."""
|
|
# The proxy here doesn't exist: a call to it fails at once rather than
|
|
# waiting out an outage's backoff (L5).
|
|
monkeypatch.setattr("nisse_dev.llm.OUTAGE_BACKOFF_S", ())
|
|
monkeypatch.setattr(
|
|
"nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing"
|
|
)
|
|
monkeypatch.setattr(
|
|
"nisse_dev.loop.build.GiteaClient.clone_url",
|
|
lambda self, repo: origins if isinstance(origins, str) else origins[repo],
|
|
)
|
|
monkeypatch.setattr(
|
|
"nisse_dev.loop.build.ModelCatalog.discover",
|
|
classmethod(lambda cls, *a, **kw: catalog or TEST_CATALOG),
|
|
)
|
|
monkeypatch.setattr(
|
|
"nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport()
|
|
)
|
|
|
|
|
|
def build_offline(
|
|
tmp_path: Path, origin: Path, monkeypatch: Any, config: AgentConfig | None = None
|
|
) -> RunningLoop:
|
|
offline(monkeypatch, str(origin))
|
|
return build_loop_agent(config or config_for(tmp_path))
|
|
|
|
|
|
def test_the_whole_graph_assembles(tmp_path: Path, origin: Path, monkeypatch) -> None:
|
|
running = build_offline(tmp_path, origin, monkeypatch)
|
|
loop = running.loop
|
|
try:
|
|
assert running.account == "nisse-dev-testing"
|
|
[project] = loop._projects.values()
|
|
assert project.repo == "don/demo"
|
|
assert (tmp_path / "work" / "README.md").exists() # cloned on the build host
|
|
assert project.milestones is not None
|
|
assert loop._driver is not None
|
|
assert loop._reader is not None # plain-language go-aheads (L5)
|
|
assert loop._helpers is not None # delegation and ask_expert (L9)
|
|
finally:
|
|
running.stop()
|
|
|
|
|
|
def test_start_announces_the_account_and_projects(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
running = build_offline(tmp_path, origin, monkeypatch)
|
|
running.loop.start = lambda: None # type: ignore[method-assign] # no polling
|
|
running.start()
|
|
readiness = running.transport.sent[-1] # type: ignore[attr-defined]
|
|
assert "nisse-dev up as nisse-dev-testing on don/demo" in readiness
|
|
running.stop()
|
|
|
|
|
|
def test_stop_closes_everything_and_releases_the_wait(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
running = build_offline(tmp_path, origin, monkeypatch)
|
|
running.stop()
|
|
|
|
assert running.transport.closed # type: ignore[attr-defined]
|
|
running.wait() # returns immediately rather than hanging
|
|
assert running.store.closed
|
|
|
|
|
|
def test_delegation_uses_the_passthrough_route(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
# L9: delegations run vendor-native; the normalized route is for the
|
|
# places where fidelity is cheap.
|
|
running = build_offline(tmp_path, origin, monkeypatch)
|
|
helpers = running.loop._helpers
|
|
assert helpers is not None
|
|
helper = helpers.delegate_client("claude-haiku-4-5")
|
|
assert isinstance(helper._inner, AnthropicPassthroughClient) # type: ignore[attr-defined]
|
|
assert isinstance(running.loop._driver._client._inner, LiteLLMClient) # type: ignore[attr-defined]
|
|
running.stop()
|
|
|
|
|
|
def test_httpx_clients_are_not_shared_between_routes(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
running = build_offline(tmp_path, origin, monkeypatch)
|
|
helpers = running.loop._helpers
|
|
assert helpers is not None
|
|
normalized = running.loop._driver._client._inner._http # type: ignore[attr-defined]
|
|
passthrough = helpers.delegate_client("claude-haiku-4-5")._inner._http # type: ignore[attr-defined]
|
|
assert isinstance(normalized, httpx.Client)
|
|
assert normalized is not passthrough # different base URLs and headers
|
|
running.stop()
|
|
|
|
|
|
def test_delegations_record_what_the_alias_resolved_to(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
running = build_offline(tmp_path, origin, monkeypatch)
|
|
helpers = running.loop._helpers
|
|
assert helpers is not None and helpers.backing is not None
|
|
assert helpers.backing("claude-haiku-4-5") == "anthropic/claude-haiku-4-5"
|
|
running.stop()
|
|
|
|
|
|
def test_the_configured_roles_are_what_items_pin(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
running = build_offline(tmp_path, origin, monkeypatch)
|
|
pinned = running.loop._roles
|
|
assert pinned == running.config.roles.pinned()
|
|
assert set(pinned) == {"driver", "helper", "expert", "classifier"}
|
|
running.stop()
|
|
|
|
|
|
# --- startup model discovery (L9) ------------------------------------
|
|
|
|
|
|
def test_a_typod_role_refuses_to_boot(tmp_path: Path, monkeypatch) -> None:
|
|
# Misconfiguration should cost a second at startup, not be discovered ten
|
|
# minutes into a milestone.
|
|
config = config_for(tmp_path, roles__driver="clause-opus-5")
|
|
monkeypatch.setattr(
|
|
"nisse_dev.loop.build.ModelCatalog.discover",
|
|
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
|
)
|
|
with pytest.raises(UnknownModel, match="driver='clause-opus-5'"):
|
|
discover_models(config, OpLog("test"))
|
|
|
|
|
|
def test_a_role_this_key_cannot_reach_refuses_to_boot(
|
|
tmp_path: Path, monkeypatch
|
|
) -> None:
|
|
# /model/info is scoped per key, so the same check catches a role pointed
|
|
# at a real model that this token has no access to.
|
|
config = config_for(tmp_path, roles__helper="local-fast")
|
|
monkeypatch.setattr(
|
|
"nisse_dev.loop.build.ModelCatalog.discover",
|
|
classmethod(lambda cls, *a, **kw: TEST_CATALOG),
|
|
)
|
|
with pytest.raises(UnknownModel, match="helper='local-fast'"):
|
|
discover_models(config, OpLog("test"))
|
|
|
|
|
|
def test_a_tool_less_model_warns_but_still_boots(tmp_path: Path, monkeypatch) -> None:
|
|
# L9 expects weak local models to be accommodated, and models LiteLLM
|
|
# declares tool-less are observed driving tool loops anyway.
|
|
import io
|
|
|
|
tool_less = ModelCatalog.from_payload(
|
|
{
|
|
"data": [
|
|
{
|
|
"model_name": alias,
|
|
"litellm_params": {"model": "openai/qwen3-8b-awq"},
|
|
"model_info": {
|
|
"litellm_provider": "openai",
|
|
"supports_function_calling": False,
|
|
"max_input_tokens": 32768,
|
|
},
|
|
}
|
|
for alias in ("claude-opus-5", "claude-haiku-4-5")
|
|
]
|
|
}
|
|
)
|
|
monkeypatch.setattr(
|
|
"nisse_dev.loop.build.ModelCatalog.discover",
|
|
classmethod(lambda cls, *a, **kw: tool_less),
|
|
)
|
|
stream = io.StringIO()
|
|
discover_models(config_for(tmp_path), OpLog("test", stream))
|
|
|
|
logged = stream.getvalue()
|
|
assert "model_tools_unsupported" in logged
|
|
assert '"role": "driver"' in logged
|
|
assert '"role": "expert"' not in logged # ask_expert drives no tools
|
|
|
|
|
|
def test_tuning_reaches_every_component(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
from dataclasses import replace
|
|
|
|
from nisse_dev.config import (
|
|
DelegationTuning,
|
|
DriverTuning,
|
|
ExpertTuning,
|
|
LoopTuning,
|
|
ToolsTuning,
|
|
Tuning,
|
|
)
|
|
|
|
config = replace(
|
|
config_for(tmp_path),
|
|
max_output_tokens=8192,
|
|
tuning=Tuning(
|
|
driver=DriverTuning(max_turns=40, failure_limit=3),
|
|
expert=ExpertTuning(consult_budget=5),
|
|
delegation=DelegationTuning(
|
|
max_turns=12, max_tokens=50_000, wall_clock_s=60
|
|
),
|
|
tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000),
|
|
loop=LoopTuning(bug_grace_s=120, general_idle_s=900),
|
|
),
|
|
)
|
|
running = build_offline(tmp_path, origin, monkeypatch, config)
|
|
loop = running.loop
|
|
driver = loop._driver
|
|
helpers = loop._helpers
|
|
assert helpers is not None
|
|
[project] = loop._projects.values()
|
|
|
|
assert (driver._max_turns, driver._failure_limit) == (40, 3)
|
|
assert helpers.expert_budget == 5
|
|
assert (helpers.caps.turns, helpers.caps.tokens) == (12, 50_000)
|
|
assert helpers.caps.wall_clock_s == 60
|
|
assert project.workspace is not None
|
|
assert project.workspace._timeout_s == 300
|
|
assert project.workspace._max_output_chars == 2000
|
|
assert (loop._grace, loop._idle) == (120, 900)
|
|
client = driver._client
|
|
assert client._inner._max_tokens == 8192 # type: ignore[attr-defined]
|
|
assert client._max_output_tokens == 8192 # type: ignore[attr-defined]
|
|
helper = helpers.delegate_client("claude-haiku-4-5")
|
|
assert helper._inner._max_tokens == 8192 # type: ignore[attr-defined]
|
|
# tool_choice is asked of exactly the models LiteLLM declares enforce it.
|
|
assert client._enforces_tool_choice("claude-opus-5") is False # type: ignore[attr-defined]
|
|
running.stop()
|
|
|
|
|
|
def test_each_project_gets_its_own_clone_and_secrets(
|
|
tmp_path: Path, origin: Path, monkeypatch
|
|
) -> None:
|
|
"""D8: a second project is only a second entry."""
|
|
second = tmp_path / "second"
|
|
config = config_for(
|
|
tmp_path,
|
|
projects__other={
|
|
"repo": "don/other",
|
|
"workspace": str(tmp_path / "other-work"),
|
|
"build_host": {
|
|
"address": HOST.address,
|
|
"user": HOST.user,
|
|
"port": str(HOST.port),
|
|
"key": str(HOST._key),
|
|
},
|
|
"labels": ["discuss"],
|
|
"secrets": {
|
|
"API_KEY": {"value": "${TEST_LLM_TOKEN}", "visible": "commands"}
|
|
},
|
|
},
|
|
)
|
|
git("clone", "-q", "--bare", str(origin), str(second))
|
|
offline(monkeypatch, {"don/demo": str(origin), "don/other": str(second)})
|
|
running = build_loop_agent(config)
|
|
try:
|
|
projects = running.loop._projects
|
|
assert set(projects) == {"greet", "other"}
|
|
assert (tmp_path / "work" / "README.md").exists()
|
|
assert (tmp_path / "other-work" / "README.md").exists()
|
|
assert projects["other"].labels == ("discuss",)
|
|
assert projects["other"].workspace._secrets == {"API_KEY": "sk-x"}
|
|
assert projects["greet"].workspace._secrets == {}
|
|
finally:
|
|
running.stop()
|