"""The composition root. These tests exist because two components — the delegation harness and the comment poller — were fully built, fully tested, and reachable from nothing but their own tests. Unit tests that construct a component directly cannot catch that; only assembly can.""" import subprocess from pathlib import Path from typing import Any import httpx import pytest from test_config import SECRETS, P, tree from nisse_dev.catalog import ModelCatalog, UnknownModel from nisse_dev.config import AgentConfig from nisse_dev.execution import SSHHost from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient from nisse_dev.loop.build import RunningLoop, build_loop_agent, discover_models from nisse_dev.loop.build import prepare_workspace as prepare from nisse_dev.oplog import OpLog # What the test roles resolve to, as LiteLLM would report them. TEST_CATALOG = ModelCatalog.from_payload( { "data": [ { "model_name": alias, "litellm_params": {"model": f"anthropic/{alias}"}, "model_info": { "litellm_provider": "anthropic", "supports_function_calling": True, "max_input_tokens": 200000, "max_output_tokens": 64000, }, } for alias in ("claude-opus-5", "claude-haiku-4-5") ] } ) # The build VM for every test here: the session's throwaway sshd. HOST: SSHHost @pytest.fixture(autouse=True) def _build_host(ssh_host: SSHHost) -> None: global HOST HOST = ssh_host def config_for(tmp_path: Path, **changes: str) -> AgentConfig: return AgentConfig.from_tree( tree( state_db=str(tmp_path / "state.db"), **{ f"{P}workspace": str(tmp_path / "work"), f"{P}build_host__address": HOST.address, f"{P}build_host__user": HOST.user, f"{P}build_host__port": str(HOST.port), f"{P}build_host__key": str(HOST._key), }, **changes, ), SECRETS, ) def git(*args: str) -> None: subprocess.run(["git", *args], check=True, capture_output=True) @pytest.fixture def origin(tmp_path: Path) -> Path: """A bare repo standing in for the project on the forge.""" remote = tmp_path / "origin.git" git("init", "-q", "--bare", "-b", "main", str(remote)) seed = tmp_path / "seed" git("init", "-q", "-b", "main", str(seed)) (seed / "README.md").write_text("project\n") git("-C", str(seed), "add", "-A") git( "-C", str(seed), "-c", "user.name=s", "-c", "user.email=s@t", "commit", "-qm", "s", ) git("-C", str(seed), "push", "-q", str(remote), "HEAD:refs/heads/main") return remote def prepare_workspace(config: AgentConfig, origin: Path) -> Any: [project] = config.projects return prepare(HOST, config, "nisse-dev-testing", str(origin), None, project) def test_workspace_is_cloned_on_the_build_host_when_absent( tmp_path: Path, origin: Path ) -> None: config = config_for(tmp_path) repo = prepare_workspace(config, origin) workspace = config.projects[0].workspace assert (workspace / "README.md").exists() assert repo.current_branch() == "main" # The clone's configuration holds the plain URL it was given (#89). assert str(origin) in (workspace / ".git" / "config").read_text() def test_workspace_is_reused_when_already_cloned(tmp_path: Path, origin: Path) -> None: config = config_for(tmp_path) workspace = config.projects[0].workspace prepare_workspace(config, origin) (workspace / "local-work.txt").write_text("in progress") prepare_workspace(config, origin) # Re-cloning would silently destroy uncommitted work. assert (workspace / "local-work.txt").exists() class FakeTransport: def __init__(self) -> None: self.sent: list[str] = [] self.closed = False def post(self, text: str, thread: str | None = None) -> str: self.sent.append(text) return f"$event{len(self.sent)}" def on_message(self, callback: Any) -> None: self.callback = callback def close(self) -> None: self.closed = True def offline(monkeypatch: Any, origins: dict[str, str] | str, catalog=None) -> None: """Only the network boundaries faked: Gitea, LiteLLM's model list, Matrix.""" # The proxy here doesn't exist: a call to it fails at once rather than # waiting out an outage's backoff (L5). monkeypatch.setattr("nisse_dev.llm.OUTAGE_BACKOFF_S", ()) monkeypatch.setattr( "nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing" ) monkeypatch.setattr( "nisse_dev.loop.build.GiteaClient.clone_url", lambda self, repo: origins if isinstance(origins, str) else origins[repo], ) monkeypatch.setattr( "nisse_dev.loop.build.ModelCatalog.discover", classmethod(lambda cls, *a, **kw: catalog or TEST_CATALOG), ) monkeypatch.setattr( "nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport() ) def build_offline( tmp_path: Path, origin: Path, monkeypatch: Any, config: AgentConfig | None = None ) -> RunningLoop: offline(monkeypatch, str(origin)) return build_loop_agent(config or config_for(tmp_path)) def test_the_whole_graph_assembles(tmp_path: Path, origin: Path, monkeypatch) -> None: running = build_offline(tmp_path, origin, monkeypatch) loop = running.loop try: assert running.account == "nisse-dev-testing" [project] = loop._projects.values() assert project.repo == "don/demo" assert (tmp_path / "work" / "README.md").exists() # cloned on the build host assert project.milestones is not None assert loop._driver is not None assert loop._reader is not None # plain-language go-aheads (L5) assert loop._helpers is not None # delegation and ask_expert (L9) finally: running.stop() def test_start_announces_the_account_and_projects( tmp_path: Path, origin: Path, monkeypatch ) -> None: running = build_offline(tmp_path, origin, monkeypatch) running.loop.start = lambda: None # type: ignore[method-assign] # no polling running.start() readiness = running.transport.sent[-1] # type: ignore[attr-defined] assert "nisse-dev up as nisse-dev-testing on don/demo" in readiness running.stop() def test_stop_closes_everything_and_releases_the_wait( tmp_path: Path, origin: Path, monkeypatch ) -> None: running = build_offline(tmp_path, origin, monkeypatch) running.stop() assert running.transport.closed # type: ignore[attr-defined] running.wait() # returns immediately rather than hanging assert running.store.closed def test_delegation_uses_the_passthrough_route( tmp_path: Path, origin: Path, monkeypatch ) -> None: # L9: delegations run vendor-native; the normalized route is for the # places where fidelity is cheap. running = build_offline(tmp_path, origin, monkeypatch) helpers = running.loop._helpers assert helpers is not None helper = helpers.delegate_client("claude-haiku-4-5") assert isinstance(helper._inner, AnthropicPassthroughClient) # type: ignore[attr-defined] assert isinstance(running.loop._driver._client._inner, LiteLLMClient) # type: ignore[attr-defined] running.stop() def test_httpx_clients_are_not_shared_between_routes( tmp_path: Path, origin: Path, monkeypatch ) -> None: running = build_offline(tmp_path, origin, monkeypatch) helpers = running.loop._helpers assert helpers is not None normalized = running.loop._driver._client._inner._http # type: ignore[attr-defined] passthrough = helpers.delegate_client("claude-haiku-4-5")._inner._http # type: ignore[attr-defined] assert isinstance(normalized, httpx.Client) assert normalized is not passthrough # different base URLs and headers running.stop() def test_delegations_record_what_the_alias_resolved_to( tmp_path: Path, origin: Path, monkeypatch ) -> None: running = build_offline(tmp_path, origin, monkeypatch) helpers = running.loop._helpers assert helpers is not None and helpers.backing is not None assert helpers.backing("claude-haiku-4-5") == "anthropic/claude-haiku-4-5" running.stop() def test_the_configured_roles_are_what_items_pin( tmp_path: Path, origin: Path, monkeypatch ) -> None: running = build_offline(tmp_path, origin, monkeypatch) pinned = running.loop._roles assert pinned == running.config.roles.pinned() assert set(pinned) == {"driver", "helper", "expert", "classifier"} running.stop() # --- startup model discovery (L9) ------------------------------------ def test_a_typod_role_refuses_to_boot(tmp_path: Path, monkeypatch) -> None: # Misconfiguration should cost a second at startup, not be discovered ten # minutes into a milestone. config = config_for(tmp_path, roles__driver="clause-opus-5") monkeypatch.setattr( "nisse_dev.loop.build.ModelCatalog.discover", classmethod(lambda cls, *a, **kw: TEST_CATALOG), ) with pytest.raises(UnknownModel, match="driver='clause-opus-5'"): discover_models(config, OpLog("test")) def test_a_role_this_key_cannot_reach_refuses_to_boot( tmp_path: Path, monkeypatch ) -> None: # /model/info is scoped per key, so the same check catches a role pointed # at a real model that this token has no access to. config = config_for(tmp_path, roles__helper="local-fast") monkeypatch.setattr( "nisse_dev.loop.build.ModelCatalog.discover", classmethod(lambda cls, *a, **kw: TEST_CATALOG), ) with pytest.raises(UnknownModel, match="helper='local-fast'"): discover_models(config, OpLog("test")) def test_a_tool_less_model_warns_but_still_boots(tmp_path: Path, monkeypatch) -> None: # L9 expects weak local models to be accommodated, and models LiteLLM # declares tool-less are observed driving tool loops anyway. import io tool_less = ModelCatalog.from_payload( { "data": [ { "model_name": alias, "litellm_params": {"model": "openai/qwen3-8b-awq"}, "model_info": { "litellm_provider": "openai", "supports_function_calling": False, "max_input_tokens": 32768, }, } for alias in ("claude-opus-5", "claude-haiku-4-5") ] } ) monkeypatch.setattr( "nisse_dev.loop.build.ModelCatalog.discover", classmethod(lambda cls, *a, **kw: tool_less), ) stream = io.StringIO() discover_models(config_for(tmp_path), OpLog("test", stream)) logged = stream.getvalue() assert "model_tools_unsupported" in logged assert '"role": "driver"' in logged assert '"role": "expert"' not in logged # ask_expert drives no tools def test_tuning_reaches_every_component( tmp_path: Path, origin: Path, monkeypatch ) -> None: from dataclasses import replace from nisse_dev.config import ( DelegationTuning, DriverTuning, ExpertTuning, LoopTuning, ToolsTuning, Tuning, ) config = replace( config_for(tmp_path), max_output_tokens=8192, tuning=Tuning( driver=DriverTuning(max_turns=40, failure_limit=3), expert=ExpertTuning(consult_budget=5), delegation=DelegationTuning( max_turns=12, max_tokens=50_000, wall_clock_s=60 ), tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000), loop=LoopTuning(bug_grace_s=120, general_idle_s=900), ), ) running = build_offline(tmp_path, origin, monkeypatch, config) loop = running.loop driver = loop._driver helpers = loop._helpers assert helpers is not None [project] = loop._projects.values() assert (driver._max_turns, driver._failure_limit) == (40, 3) assert helpers.expert_budget == 5 assert (helpers.caps.turns, helpers.caps.tokens) == (12, 50_000) assert helpers.caps.wall_clock_s == 60 assert project.workspace is not None assert project.workspace._timeout_s == 300 assert project.workspace._max_output_chars == 2000 assert (loop._grace, loop._idle) == (120, 900) client = driver._client assert client._inner._max_tokens == 8192 # type: ignore[attr-defined] assert client._max_output_tokens == 8192 # type: ignore[attr-defined] helper = helpers.delegate_client("claude-haiku-4-5") assert helper._inner._max_tokens == 8192 # type: ignore[attr-defined] # tool_choice is asked of exactly the models LiteLLM declares enforce it. assert client._enforces_tool_choice("claude-opus-5") is False # type: ignore[attr-defined] running.stop() def test_each_project_gets_its_own_clone_and_secrets( tmp_path: Path, origin: Path, monkeypatch ) -> None: """D8: a second project is only a second entry.""" second = tmp_path / "second" config = config_for( tmp_path, projects__other={ "repo": "don/other", "workspace": str(tmp_path / "other-work"), "build_host": { "address": HOST.address, "user": HOST.user, "port": str(HOST.port), "key": str(HOST._key), }, "labels": ["discuss"], "secrets": { "API_KEY": {"value": "${TEST_LLM_TOKEN}", "visible": "commands"} }, }, ) git("clone", "-q", "--bare", str(origin), str(second)) offline(monkeypatch, {"don/demo": str(origin), "don/other": str(second)}) running = build_loop_agent(config) try: projects = running.loop._projects assert set(projects) == {"greet", "other"} assert (tmp_path / "work" / "README.md").exists() assert (tmp_path / "other-work" / "README.md").exists() assert projects["other"].labels == ("discuss",) assert projects["other"].workspace._secrets == {"API_KEY": "sk-x"} assert projects["greet"].workspace._secrets == {} finally: running.stop()