nisse/nisse-dev/tests/test_agent_build.py

"""The composition root. These tests exist because two components — the
delegation harness and the comment poller — were fully built, fully tested,
and reachable from nothing but their own tests. Unit tests that construct a
component directly cannot catch that; only assembly can."""

import subprocess
from pathlib import Path
from typing import Any

import httpx
import pytest
from test_config import SECRETS, P, tree

from nisse_dev.agent import build_agent, discover_models, prepare_workspace
from nisse_dev.catalog import ModelCatalog, UnknownModel
from nisse_dev.config import AgentConfig
from nisse_dev.execution import SSHHost
from nisse_dev.oplog import OpLog

# What the test roles resolve to, as LiteLLM would report them.
TEST_CATALOG = ModelCatalog.from_payload(
    {
        "data": [
            {
                "model_name": alias,
                "litellm_params": {"model": f"anthropic/{alias}"},
                "model_info": {
                    "litellm_provider": "anthropic",
                    "supports_function_calling": True,
                    "max_input_tokens": 200000,
                    "max_output_tokens": 64000,
                },
            }
            for alias in ("claude-opus-5", "claude-haiku-4-5")
        ]
    }
)


# The build VM for every test here: the session's throwaway sshd.
HOST: SSHHost


@pytest.fixture(autouse=True)
def _build_host(ssh_host: SSHHost) -> None:
    global HOST
    HOST = ssh_host


def config_for(tmp_path: Path, **changes: str) -> AgentConfig:
    return AgentConfig.from_tree(
        tree(
            state_db=str(tmp_path / "state.db"),
            **{
                f"{P}workspace": str(tmp_path / "work"),
                f"{P}build_host__address": HOST.address,
                f"{P}build_host__user": HOST.user,
                f"{P}build_host__port": str(HOST.port),
                f"{P}build_host__key": str(HOST._key),
            },
            **changes,
        ),
        SECRETS,
    )


def git(*args: str) -> None:
    subprocess.run(["git", *args], check=True, capture_output=True)


@pytest.fixture
def origin(tmp_path: Path) -> Path:
    """A bare repo standing in for the project on the forge."""
    remote = tmp_path / "origin.git"
    git("init", "-q", "--bare", "-b", "main", str(remote))
    seed = tmp_path / "seed"
    git("init", "-q", "-b", "main", str(seed))
    (seed / "README.md").write_text("project\n")
    git("-C", str(seed), "add", "-A")
    git(
        "-C",
        str(seed),
        "-c",
        "user.name=s",
        "-c",
        "user.email=s@t",
        "commit",
        "-qm",
        "s",
    )
    git("-C", str(seed), "push", "-q", str(remote), "HEAD:refs/heads/main")
    return remote


def test_workspace_is_cloned_on_the_build_host_when_absent(
    tmp_path: Path, origin: Path
) -> None:
    config = config_for(tmp_path)
    repo = prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None)

    assert (config.workspace / "README.md").exists()
    assert repo.current_branch() == "main"
    # The clone's configuration holds the plain URL it was given (#89).
    assert str(origin) in (config.workspace / ".git" / "config").read_text()


def test_workspace_is_reused_when_already_cloned(tmp_path: Path, origin: Path) -> None:
    config = config_for(tmp_path)
    prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None)
    (config.workspace / "local-work.txt").write_text("in progress")

    prepare_workspace(HOST, config, "nisse-dev-testing", str(origin), None)

    # Re-cloning would silently destroy uncommitted work.
    assert (config.workspace / "local-work.txt").exists()


class FakeTransport:
    def __init__(self) -> None:
        self.sent: list[str] = []
        self.started = False
        self.closed = False

    def send_message(self, text: str) -> None:
        self.sent.append(text)

    def send_approval_request(self, text: str) -> None:
        self.sent.append(text)

    def ask_question(self, text: str) -> None:
        self.sent.append(text)

    def on_message(self, callback: Any) -> None:
        self.callback = callback

    def start(self) -> None:
        self.started = True

    def close(self) -> None:
        self.closed = True


def build_offline(tmp_path: Path, origin: Path, monkeypatch: Any) -> Any:
    """Assemble the real graph with only the network boundaries faked."""
    config = config_for(tmp_path)

    monkeypatch.setattr(
        "nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"
    )
    monkeypatch.setattr(
        "nisse_dev.agent.ModelCatalog.discover",
        classmethod(lambda cls, *a, **kw: TEST_CATALOG),
    )
    monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport())
    monkeypatch.setattr(
        "nisse_dev.agent.GiteaClient.clone_url", lambda self, repo: str(origin)
    )
    return build_agent(config)


def test_the_whole_graph_assembles(tmp_path: Path, origin: Path, monkeypatch) -> None:
    agent = build_offline(tmp_path, origin, monkeypatch)
    orchestrator = agent.orchestrator

    # Every component that was previously unreachable is now connected.
    assert orchestrator._executor is not None
    assert orchestrator._publisher is not None
    assert orchestrator._board is not None
    assert orchestrator._delegate is not None
    assert orchestrator._poller_factory is not None
    assert orchestrator._executor._delegate is not None  # L9 escalation path
    assert orchestrator._executor._expert is not None  # L9 consult path
    agent.stop()


def test_start_announces_where_it_resumed(
    tmp_path: Path, origin: Path, monkeypatch
) -> None:
    agent = build_offline(tmp_path, origin, monkeypatch)
    agent.start()

    assert agent.transport.started
    readiness = agent.transport.sent[-1]
    assert "nisse-dev up as nisse-dev-testing" in readiness
    assert "greet" in readiness  # the project
    assert "claude-opus-5" in readiness  # the pinned planner
    agent.stop()


def test_stop_closes_everything_and_releases_the_wait(
    tmp_path: Path, origin: Path, monkeypatch
) -> None:
    agent = build_offline(tmp_path, origin, monkeypatch)
    agent.start()
    agent.stop()

    assert agent.transport.closed
    agent.wait()  # returns immediately rather than hanging
    assert agent.store.closed


def test_poller_factory_ignores_the_agents_own_comments(
    tmp_path: Path, origin: Path, monkeypatch
) -> None:
    # Without this the agent answers its own PR replies in a loop.
    agent = build_offline(tmp_path, origin, monkeypatch)
    poller = agent.orchestrator._poller_factory(lambda c: None)
    assert "nisse-dev-testing" in poller._ignore
    agent.stop()


def test_delegation_uses_the_passthrough_route(
    tmp_path: Path, origin: Path, monkeypatch
) -> None:
    # L9: delegations run vendor-native; the normalized route is for the
    # places where fidelity is cheap.
    from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient

    agent = build_offline(tmp_path, origin, monkeypatch)
    delegation = agent.orchestrator._delegate
    assert isinstance(delegation._client._inner, AnthropicPassthroughClient)
    assert isinstance(agent.orchestrator._executor._client._inner, LiteLLMClient)
    agent.stop()


def test_httpx_clients_are_not_shared_between_routes(
    tmp_path: Path, origin: Path, monkeypatch
) -> None:
    agent = build_offline(tmp_path, origin, monkeypatch)
    normalized = agent.orchestrator._executor._client._inner._http
    passthrough = agent.orchestrator._delegate._client._inner._http
    assert isinstance(normalized, httpx.Client)
    assert normalized is not passthrough  # different base URLs and headers
    agent.stop()


# --- startup model discovery (L9) ------------------------------------


def test_a_typod_role_refuses_to_boot(tmp_path: Path, monkeypatch) -> None:
    # Misconfiguration should cost a second at startup, not be discovered ten
    # minutes into a milestone.
    config = config_for(tmp_path, roles__planner="clause-opus-5")
    monkeypatch.setattr(
        "nisse_dev.agent.ModelCatalog.discover",
        classmethod(lambda cls, *a, **kw: TEST_CATALOG),
    )
    with pytest.raises(UnknownModel, match="planner='clause-opus-5'"):
        discover_models(config, OpLog("test"))


def test_a_role_this_key_cannot_reach_refuses_to_boot(
    tmp_path: Path, monkeypatch
) -> None:
    # /model/info is scoped per key, so the same check catches a role pointed
    # at a real model that this token has no access to.
    config = config_for(tmp_path, roles__executor="local-fast")
    monkeypatch.setattr(
        "nisse_dev.agent.ModelCatalog.discover",
        classmethod(lambda cls, *a, **kw: TEST_CATALOG),
    )
    with pytest.raises(UnknownModel, match="executor='local-fast'"):
        discover_models(config, OpLog("test"))


def test_a_tool_less_model_warns_but_still_boots(tmp_path: Path, monkeypatch) -> None:
    # L9 expects weak local models to be accommodated, and models LiteLLM
    # declares tool-less are observed driving tool loops anyway.
    import io

    tool_less = ModelCatalog.from_payload(
        {
            "data": [
                {
                    "model_name": alias,
                    "litellm_params": {"model": "openai/qwen3-8b-awq"},
                    "model_info": {
                        "litellm_provider": "openai",
                        "supports_function_calling": False,
                        "max_input_tokens": 32768,
                    },
                }
                for alias in ("claude-opus-5", "claude-haiku-4-5")
            ]
        }
    )
    monkeypatch.setattr(
        "nisse_dev.agent.ModelCatalog.discover",
        classmethod(lambda cls, *a, **kw: tool_less),
    )
    stream = io.StringIO()
    discover_models(config_for(tmp_path), OpLog("test", stream))

    logged = stream.getvalue()
    assert "model_tools_unsupported" in logged
    assert '"role": "executor"' in logged
    assert '"role": "expert"' not in logged  # ask_expert drives no tools


def test_the_resolved_model_is_pinned_at_task_start(
    tmp_path: Path, origin: Path, monkeypatch
) -> None:
    agent = build_offline(tmp_path, origin, monkeypatch)
    # The store belongs to the worker thread, so the work has to run there.
    agent.worker.call(lambda: agent.orchestrator._begin_project("greet"))

    pinned = agent.orchestrator._data["models"]
    assert pinned["planner"] == {
        "alias": "claude-opus-5",
        "backing": "anthropic/claude-opus-5",
        "provider": "anthropic",
    }
    agent.stop()


def test_delegation_records_what_the_alias_resolved_to(
    tmp_path: Path, origin: Path, monkeypatch
) -> None:
    # L9 eval history keyed on an alias alone would average across whatever
    # that alias pointed at over time.
    agent = build_offline(tmp_path, origin, monkeypatch)
    assert agent.orchestrator._delegate._backing_model == "anthropic/claude-opus-5"
    agent.stop()


def test_tuning_reaches_every_component(
    tmp_path: Path, origin: Path, monkeypatch
) -> None:
    from dataclasses import replace

    from nisse_dev.config import (
        DelegationTuning,
        ExecutorTuning,
        ExpertTuning,
        ToolsTuning,
        Tuning,
    )

    config = replace(
        config_for(tmp_path),
        max_output_tokens=8192,
        tuning=Tuning(
            executor=ExecutorTuning(max_turns=40, failure_limit=3),
            expert=ExpertTuning(consult_budget=5),
            delegation=DelegationTuning(
                max_turns=12, max_tokens=50_000, wall_clock_s=60
            ),
            tools=ToolsTuning(command_timeout_s=300, max_output_chars=2000),
        ),
    )
    monkeypatch.setattr(
        "nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"
    )
    monkeypatch.setattr(
        "nisse_dev.agent.ModelCatalog.discover",
        classmethod(lambda cls, *a, **kw: TEST_CATALOG),
    )
    monkeypatch.setattr("nisse_dev.agent.MatrixTransport", lambda cfg: FakeTransport())
    monkeypatch.setattr(
        "nisse_dev.agent.GiteaClient.clone_url", lambda self, repo: str(origin)
    )
    agent = build_agent(config)
    executor = agent.orchestrator._executor
    delegation = agent.orchestrator._delegate

    assert (executor._max_turns, executor._failure_limit) == (40, 3)
    assert executor._expert._budget == 5
    assert delegation._caps.turns == 12
    assert delegation._caps.tokens == 50_000
    assert delegation._caps.wall_clock_s == 60
    assert executor._workspace._timeout_s == 300
    assert executor._workspace._max_output_chars == 2000
    assert executor._client._inner._max_tokens == 8192
    assert delegation._client._inner._max_tokens == 8192
    assert executor._client._max_output_tokens == 8192
    # tool_choice is asked of exactly the models LiteLLM declares enforce it.
    assert executor._client._enforces_tool_choice("claude-opus-5") is False
    agent.stop()


def test_the_new_loop_assembles(tmp_path: Path, origin: Path, monkeypatch) -> None:
    """`loop: new` builds the LoopAgent over the same services."""
    from nisse_dev.loop.build import build_loop_agent

    config = config_for(tmp_path, loop="new")
    monkeypatch.setattr(
        "nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing"
    )
    monkeypatch.setattr(
        "nisse_dev.loop.build.GiteaClient.clone_url", lambda self, repo: str(origin)
    )
    monkeypatch.setattr(
        "nisse_dev.agent.ModelCatalog.discover",
        classmethod(lambda cls, *a, **kw: TEST_CATALOG),
    )
    monkeypatch.setattr(
        "nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport()
    )
    running = build_loop_agent(config)
    try:
        assert running.account == "nisse-dev-testing"
        [project] = running.loop._projects.values()
        assert project.repo == config.gitea_repo
        assert (config.workspace / "README.md").exists()  # cloned on the build host
    finally:
        running.worker.call(running.store.close)
        running.worker.stop(timeout=5)


def test_each_project_gets_its_own_clone_and_secrets(
    tmp_path: Path, origin: Path, monkeypatch
) -> None:
    """D8: a second project is only a second entry."""
    from nisse_dev.loop.build import build_loop_agent

    second = tmp_path / "second"
    config = config_for(
        tmp_path,
        loop="new",
        projects__other={
            "repo": "don/other",
            "workspace": str(tmp_path / "other-work"),
            "build_host": {
                "address": HOST.address,
                "user": HOST.user,
                "port": str(HOST.port),
                "key": str(HOST._key),
            },
            "labels": ["discuss"],
            "secrets": {
                "API_KEY": {"value": "${TEST_LLM_TOKEN}", "visible": "commands"}
            },
        },
    )
    git("clone", "-q", "--bare", str(origin), str(second))
    origins = {"don/demo": str(origin), "don/other": str(second)}
    monkeypatch.setattr(
        "nisse_dev.loop.build.GiteaClient.whoami", lambda self: "nisse-dev-testing"
    )
    monkeypatch.setattr(
        "nisse_dev.loop.build.GiteaClient.clone_url", lambda self, repo: origins[repo]
    )
    monkeypatch.setattr(
        "nisse_dev.agent.ModelCatalog.discover",
        classmethod(lambda cls, *a, **kw: TEST_CATALOG),
    )
    monkeypatch.setattr(
        "nisse_dev.loop.build.MatrixTransport", lambda cfg: FakeTransport()
    )
    running = build_loop_agent(config)
    try:
        projects = running.loop._projects
        assert set(projects) == {"greet", "other"}
        assert (tmp_path / "work" / "README.md").exists()
        assert (tmp_path / "other-work" / "README.md").exists()
        assert projects["other"].labels == ("discuss",)
        assert projects["other"].workspace._secrets == {"API_KEY": "sk-x"}
        assert projects["greet"].workspace._secrets == {}
    finally:
        running.worker.call(running.store.close)
        running.worker.stop(timeout=5)