diff --git a/nisse-dev/src/nisse_dev/agent.py b/nisse-dev/src/nisse_dev/agent.py index 4ffa084..78522cf 100644 --- a/nisse-dev/src/nisse_dev/agent.py +++ b/nisse-dev/src/nisse_dev/agent.py @@ -19,7 +19,11 @@ from nisse_dev.executor import LLMExecutor from nisse_dev.expert import ExpertDesk from nisse_dev.git import GitRepo, authenticated_url from nisse_dev.gitea import Comment, GiteaClient -from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient +from nisse_dev.llm import ( + AnthropicPassthroughClient, + LiteLLMClient, + select_client, +) from nisse_dev.matrix import MatrixTransport from nisse_dev.oplog import OpLog from nisse_dev.planner import LLMPlanningAgent @@ -98,19 +102,22 @@ def build_agent( workspace = Workspace(config.workspace) normalized = LiteLLMClient(config.llm_base_url, config.llm_token) - # Delegations take the passthrough route; fidelity is cheap elsewhere (A5). + # Delegations prefer the vendor-native route; fidelity is cheap + # elsewhere (A5). Which route a given model gets is decided by + # select_client, not by where the caller happens to sit. passthrough = AnthropicPassthroughClient(config.llm_base_url, config.llm_token) transport = MatrixTransport(config.matrix) announce = notify or transport.send_message expert = ExpertDesk(normalized, store, config.roles) + delegate_model = config.roles.resolve("delegate") delegation = Delegation( - passthrough, + select_client(delegate_model, normalized, passthrough), store, workspace, git_repo, - model=config.roles.resolve("delegate"), + model=delegate_model, ) executor = LLMExecutor( normalized, diff --git a/nisse-dev/src/nisse_dev/llm.py b/nisse-dev/src/nisse_dev/llm.py index 55f6ac1..e035a98 100644 --- a/nisse-dev/src/nisse_dev/llm.py +++ b/nisse-dev/src/nisse_dev/llm.py @@ -276,3 +276,28 @@ def _post_with_rate_limit_retry( return response sleep(RATE_LIMIT_RETRY_S) raise AssertionError("unreachable") + + +# Model families we have a native renderer adapter for. A13: delegations take +# the vendor's passthrough route because a normalising middlebox is lossy +# exactly where a long-running delegation lives. +PASSTHROUGH_PREFIXES = ("claude-", "anthropic/") + + +def has_native_adapter(model: str) -> bool: + return model.startswith(PASSTHROUGH_PREFIXES) + + +def select_client( + model: str, normalized: LLMClient, passthrough: LLMClient +) -> LLMClient: + """Choose the route from the model, not from where the caller sits. + + A13 spells out the rule: use the vendor-native adapter where one exists, + and otherwise *provisionally* accept LiteLLM's normalization for that + vendor — validating it against the A16 eval set before trusting it with + delegations. Hard-wiring the passthrough would mean pointing a role at a + non-Anthropic model silently sent it Anthropic-shaped request bodies, + which is the opposite of the vendor fluidity A5 requires. + """ + return passthrough if has_native_adapter(model) else normalized diff --git a/nisse-dev/tests/test_llm.py b/nisse-dev/tests/test_llm.py index 25706f9..abcec92 100644 --- a/nisse-dev/tests/test_llm.py +++ b/nisse-dev/tests/test_llm.py @@ -11,7 +11,14 @@ from nisse_dev.canonical import ( UserMessage, ) from nisse_dev.envfile import load_env_file -from nisse_dev.llm import RATE_LIMIT_RETRY_S, LiteLLMClient, LLMError, RoleMap +from nisse_dev.llm import ( + RATE_LIMIT_RETRY_S, + LiteLLMClient, + LLMError, + RoleMap, + has_native_adapter, + select_client, +) EVENTS = [StoredEvent(id=1, ts="t", event=UserMessage(text="hi", source="m:1"))] @@ -203,3 +210,28 @@ def test_load_env_file_missing_is_noop(tmp_path: Path) -> None: env: dict[str, str] = {} load_env_file(tmp_path / "nope.env", env) assert env == {} + + +# --- route selection (A5 vendor fluidity) -------------------------------- + + +def test_anthropic_models_take_the_native_route() -> None: + normalized, passthrough = object(), object() + for model in ("claude-haiku-4-5", "claude-opus-5", "anthropic/claude-sonnet-5"): + assert select_client(model, normalized, passthrough) is passthrough + + +def test_other_vendors_take_the_normalized_route() -> None: + # A13: provisionally accept LiteLLM's normalization for a vendor whose + # renderer adapter does not exist yet, rather than sending it request + # bodies shaped for someone else. + normalized, passthrough = object(), object() + for model in ("gpt-6-astra", "local-coder", "local-fast", "llama3.2"): + assert select_client(model, normalized, passthrough) is normalized + + +def test_native_adapter_detection_is_explicit() -> None: + assert has_native_adapter("claude-opus-5") + assert not has_native_adapter("gpt-6-astra") + # A model merely mentioning the vendor is not the vendor. + assert not has_native_adapter("my-claude-clone") diff --git a/tests/integration/test_model_capability.py b/tests/integration/test_model_capability.py new file mode 100644 index 0000000..710b8eb --- /dev/null +++ b/tests/integration/test_model_capability.py @@ -0,0 +1,162 @@ +"""Capability matrix across models (spec Part H, A15). + +Part H says the thresholds that depend on local capability — A12's failure +count, A14's consult budget, A15's manifest size — must be tuned "at +build-order step 5 on the real hardware". This is the measurement that makes +that possible, rather than reasoning about model strength from reputation. + +What it measures is the thing that actually breaks: whether a model can hold +the tool-call contract our loops depend on. llama3.2 could not — it returned +`{` as an entire plan — and no amount of prompt work fixed it. + +Opt in with NISSE_CAPABILITY_MODELS, comma separated. Skipped otherwise, +because it spends real tokens across several models. +""" + +import os +from pathlib import Path + +import pytest + +from nisse_dev.canonical import ( + AssistantText, + StoredEvent, + ToolCall, + ToolResult, + UserMessage, +) +from nisse_dev.envfile import load_env_file +from nisse_dev.expert import parse_answer +from nisse_dev.llm import ( + AnthropicPassthroughClient, + LiteLLMClient, + LLMError, + select_client, +) +from nisse_dev.planner import PLANNER_TOOLS +from nisse_dev.tools import EXECUTOR_TOOLS + +load_env_file(Path(__file__).resolve().parents[2] / ".env") + +SENTINEL = "NISSE_CAPABILITY_MODELS" +MODELS = [m.strip() for m in os.environ.get(SENTINEL, "").split(",") if m.strip()] + +pytestmark = [ + pytest.mark.integration, + pytest.mark.skipif( + not MODELS, reason=f"set {SENTINEL} to a comma-separated model list" + ), +] + + +def clients() -> tuple[LiteLLMClient, AnthropicPassthroughClient]: + return LiteLLMClient.from_env(), AnthropicPassthroughClient.from_env() + + +def user(text: str) -> list[StoredEvent]: + return [StoredEvent(id=1, ts="t", event=UserMessage(text=text, source="probe"))] + + +@pytest.mark.parametrize("model", MODELS) +def test_model_answers_at_all(model: str) -> None: + normalized, passthrough = clients() + client = select_client(model, normalized, passthrough) + produced = client.complete( + user("Reply with exactly one word: pong"), None, model, system="Be terse." + ) + texts = [e.text for e in produced if isinstance(e, AssistantText)] + assert texts, f"{model} produced no text: {produced!r}" + + +@pytest.mark.parametrize("model", MODELS) +def test_model_holds_the_planner_contract(model: str) -> None: + """A4's gating is structural at the tool layer: a model that answers in + prose cannot be gated, because there is no plan to approve.""" + normalized, passthrough = clients() + client = select_client(model, normalized, passthrough) + produced = client.complete( + user("Draft an architecture plan for a command-line slug generator."), + PLANNER_TOOLS, + model, + system=( + "You must answer by calling exactly one tool: submit_plan or " + "ask_human. Plain prose is not accepted." + ), + ) + calls = [e for e in produced if isinstance(e, ToolCall)] + assert calls, f"{model} answered in prose instead of calling a tool" + assert calls[0].tool in {"submit_plan", "ask_human"} + if calls[0].tool == "submit_plan": + content = calls[0].arguments.get("content", "") + # llama3.2 emitted "{" here — structurally a tool call, substantively + # nothing. Length is a crude proxy, but it separates the two. + assert len(str(content)) > 80, ( + f"{model} submitted a degenerate plan: {content!r}" + ) + + +@pytest.mark.parametrize("model", MODELS) +def test_model_round_trips_a_tool_result(model: str) -> None: + """The executor loop feeds results back; a model that cannot consume its + own tool result cannot run a milestone.""" + normalized, passthrough = clients() + client = select_client(model, normalized, passthrough) + events = user("List the files in the current directory using your tools.") + produced = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.") + calls = [e for e in produced if isinstance(e, ToolCall)] + assert calls, f"{model} made no tool call" + + next_id = 2 + for event in produced: + events.append(StoredEvent(id=next_id, ts="t", event=event)) + next_id += 1 + events.append( + StoredEvent( + id=next_id, + ts="t", + event=ToolResult( + call_id=calls[0].call_id, result="app.py\ntests/", success=True + ), + ) + ) + follow_up = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.") + assert follow_up, f"{model} produced nothing after its tool result" + + +@pytest.mark.parametrize("model", MODELS) +def test_model_holds_the_prompt_level_json_contract(model: str) -> None: + """A14 contracts structured output at the prompt level because vendors + differ on JSON enforcement. This measures whether that holds per model.""" + normalized, passthrough = clients() + client = select_client(model, normalized, passthrough) + produced = client.complete( + user("Is SQLite sufficient for a single-process agent's task state?"), + None, + model, + system=( + "Reply with a single JSON object and nothing else: " + '{"answer": "...", "confidence": "high"|"medium"|"low", ' + '"caveats": "..."}' + ), + ) + text = "\n".join(e.text for e in produced if isinstance(e, AssistantText)) + assert parse_answer(text) is not None, ( + f"{model} did not produce parseable JSON: {text[:200]!r}" + ) + + +@pytest.mark.parametrize("model", MODELS) +def test_route_selection_matches_the_model(model: str) -> None: + """A non-Anthropic model must not be sent Anthropic-shaped bodies. This + was hardcoded until gpt-6-astra made it observable.""" + normalized, passthrough = clients() + chosen = select_client(model, normalized, passthrough) + if model.startswith(("claude-", "anthropic/")): + assert chosen is passthrough + else: + assert chosen is normalized + # And the chosen route must actually work for this model. + try: + chosen.complete(user("say pong"), None, model, system="Be terse.") + except LLMError as exc: + pytest.fail(f"{model} failed on its selected route: {exc}")