Choose the LLM route by model, and add a capability matrix

The arrival of non-Anthropic models made a latent bug observable: the
delegate's client was hardcoded to the Anthropic passthrough, so
pointing NISSE_ROLE_DELEGATE at gpt-6-astra or a local model would
have sent Anthropic-shaped /v1/messages bodies to something that does
not speak them. A5 promises the frontier tier is swappable "as a
configuration change, not a refactor", and it was not.

select_client now picks the route from the model: the vendor-native
adapter where one exists, LiteLLM's normalization otherwise — which is
exactly what A13 prescribes for a vendor whose renderer adapter has
not been written, pending validation against the A16 eval set.

Also adds the capability matrix Part H asks for. It says the
thresholds that depend on local capability — A12's failure count,
A14's consult budget, A15's manifest size — must be tuned "on the real
hardware", and that needs measurement rather than reputation. The
harness probes each model for the things that actually break: whether
it calls a tool at all, whether the plan it submits has substance
(llama3.2 emitted "{"), whether it can consume its own tool result,
and whether prompt-level JSON holds. Opt in with
NISSE_CAPABILITY_MODELS; it spends real tokens across several models.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

nisse-dev/src/nisse_dev/agent.py

@@ -19,7 +19,11 @@ from nisse_dev.executor import LLMExecutor
 from nisse_dev.expert import ExpertDesk
 from nisse_dev.git import GitRepo, authenticated_url
 from nisse_dev.gitea import Comment, GiteaClient
-from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
+from nisse_dev.llm import (
+    AnthropicPassthroughClient,
+    LiteLLMClient,
+    select_client,
+)
 from nisse_dev.matrix import MatrixTransport
 from nisse_dev.oplog import OpLog
 from nisse_dev.planner import LLMPlanningAgent
@@ -98,19 +102,22 @@ def build_agent(
     workspace = Workspace(config.workspace)
 
     normalized = LiteLLMClient(config.llm_base_url, config.llm_token)
-    # Delegations take the passthrough route; fidelity is cheap elsewhere (A5).
+    # Delegations prefer the vendor-native route; fidelity is cheap
+    # elsewhere (A5). Which route a given model gets is decided by
+    # select_client, not by where the caller happens to sit.
     passthrough = AnthropicPassthroughClient(config.llm_base_url, config.llm_token)
 
     transport = MatrixTransport(config.matrix)
     announce = notify or transport.send_message
 
     expert = ExpertDesk(normalized, store, config.roles)
+    delegate_model = config.roles.resolve("delegate")
     delegation = Delegation(
-        passthrough,
+        select_client(delegate_model, normalized, passthrough),
         store,
         workspace,
         git_repo,
-        model=config.roles.resolve("delegate"),
+        model=delegate_model,
     )
     executor = LLMExecutor(
         normalized,

nisse-dev/src/nisse_dev/llm.py

@@ -276,3 +276,28 @@ def _post_with_rate_limit_retry(
             return response
         sleep(RATE_LIMIT_RETRY_S)
     raise AssertionError("unreachable")
+
+
+# Model families we have a native renderer adapter for. A13: delegations take
+# the vendor's passthrough route because a normalising middlebox is lossy
+# exactly where a long-running delegation lives.
+PASSTHROUGH_PREFIXES = ("claude-", "anthropic/")
+
+
+def has_native_adapter(model: str) -> bool:
+    return model.startswith(PASSTHROUGH_PREFIXES)
+
+
+def select_client(
+    model: str, normalized: LLMClient, passthrough: LLMClient
+) -> LLMClient:
+    """Choose the route from the model, not from where the caller sits.
+
+    A13 spells out the rule: use the vendor-native adapter where one exists,
+    and otherwise *provisionally* accept LiteLLM's normalization for that
+    vendor — validating it against the A16 eval set before trusting it with
+    delegations. Hard-wiring the passthrough would mean pointing a role at a
+    non-Anthropic model silently sent it Anthropic-shaped request bodies,
+    which is the opposite of the vendor fluidity A5 requires.
+    """
+    return passthrough if has_native_adapter(model) else normalized

nisse-dev/tests/test_llm.py

@@ -11,7 +11,14 @@ from nisse_dev.canonical import (
     UserMessage,
 )
 from nisse_dev.envfile import load_env_file
-from nisse_dev.llm import RATE_LIMIT_RETRY_S, LiteLLMClient, LLMError, RoleMap
+from nisse_dev.llm import (
+    RATE_LIMIT_RETRY_S,
+    LiteLLMClient,
+    LLMError,
+    RoleMap,
+    has_native_adapter,
+    select_client,
+)
 
 EVENTS = [StoredEvent(id=1, ts="t", event=UserMessage(text="hi", source="m:1"))]
 
@@ -203,3 +210,28 @@ def test_load_env_file_missing_is_noop(tmp_path: Path) -> None:
     env: dict[str, str] = {}
     load_env_file(tmp_path / "nope.env", env)
     assert env == {}
+
+
+# --- route selection (A5 vendor fluidity) --------------------------------
+
+
+def test_anthropic_models_take_the_native_route() -> None:
+    normalized, passthrough = object(), object()
+    for model in ("claude-haiku-4-5", "claude-opus-5", "anthropic/claude-sonnet-5"):
+        assert select_client(model, normalized, passthrough) is passthrough
+
+
+def test_other_vendors_take_the_normalized_route() -> None:
+    # A13: provisionally accept LiteLLM's normalization for a vendor whose
+    # renderer adapter does not exist yet, rather than sending it request
+    # bodies shaped for someone else.
+    normalized, passthrough = object(), object()
+    for model in ("gpt-6-astra", "local-coder", "local-fast", "llama3.2"):
+        assert select_client(model, normalized, passthrough) is normalized
+
+
+def test_native_adapter_detection_is_explicit() -> None:
+    assert has_native_adapter("claude-opus-5")
+    assert not has_native_adapter("gpt-6-astra")
+    # A model merely mentioning the vendor is not the vendor.
+    assert not has_native_adapter("my-claude-clone")

tests/integration/test_model_capability.py

@@ -0,0 +1,162 @@
+"""Capability matrix across models (spec Part H, A15).
+
+Part H says the thresholds that depend on local capability — A12's failure
+count, A14's consult budget, A15's manifest size — must be tuned "at
+build-order step 5 on the real hardware". This is the measurement that makes
+that possible, rather than reasoning about model strength from reputation.
+
+What it measures is the thing that actually breaks: whether a model can hold
+the tool-call contract our loops depend on. llama3.2 could not — it returned
+`{` as an entire plan — and no amount of prompt work fixed it.
+
+Opt in with NISSE_CAPABILITY_MODELS, comma separated. Skipped otherwise,
+because it spends real tokens across several models.
+"""
+
+import os
+from pathlib import Path
+
+import pytest
+
+from nisse_dev.canonical import (
+    AssistantText,
+    StoredEvent,
+    ToolCall,
+    ToolResult,
+    UserMessage,
+)
+from nisse_dev.envfile import load_env_file
+from nisse_dev.expert import parse_answer
+from nisse_dev.llm import (
+    AnthropicPassthroughClient,
+    LiteLLMClient,
+    LLMError,
+    select_client,
+)
+from nisse_dev.planner import PLANNER_TOOLS
+from nisse_dev.tools import EXECUTOR_TOOLS
+
+load_env_file(Path(__file__).resolve().parents[2] / ".env")
+
+SENTINEL = "NISSE_CAPABILITY_MODELS"
+MODELS = [m.strip() for m in os.environ.get(SENTINEL, "").split(",") if m.strip()]
+
+pytestmark = [
+    pytest.mark.integration,
+    pytest.mark.skipif(
+        not MODELS, reason=f"set {SENTINEL} to a comma-separated model list"
+    ),
+]
+
+
+def clients() -> tuple[LiteLLMClient, AnthropicPassthroughClient]:
+    return LiteLLMClient.from_env(), AnthropicPassthroughClient.from_env()
+
+
+def user(text: str) -> list[StoredEvent]:
+    return [StoredEvent(id=1, ts="t", event=UserMessage(text=text, source="probe"))]
+
+
+@pytest.mark.parametrize("model", MODELS)
+def test_model_answers_at_all(model: str) -> None:
+    normalized, passthrough = clients()
+    client = select_client(model, normalized, passthrough)
+    produced = client.complete(
+        user("Reply with exactly one word: pong"), None, model, system="Be terse."
+    )
+    texts = [e.text for e in produced if isinstance(e, AssistantText)]
+    assert texts, f"{model} produced no text: {produced!r}"
+
+
+@pytest.mark.parametrize("model", MODELS)
+def test_model_holds_the_planner_contract(model: str) -> None:
+    """A4's gating is structural at the tool layer: a model that answers in
+    prose cannot be gated, because there is no plan to approve."""
+    normalized, passthrough = clients()
+    client = select_client(model, normalized, passthrough)
+    produced = client.complete(
+        user("Draft an architecture plan for a command-line slug generator."),
+        PLANNER_TOOLS,
+        model,
+        system=(
+            "You must answer by calling exactly one tool: submit_plan or "
+            "ask_human. Plain prose is not accepted."
+        ),
+    )
+    calls = [e for e in produced if isinstance(e, ToolCall)]
+    assert calls, f"{model} answered in prose instead of calling a tool"
+    assert calls[0].tool in {"submit_plan", "ask_human"}
+    if calls[0].tool == "submit_plan":
+        content = calls[0].arguments.get("content", "")
+        # llama3.2 emitted "{" here — structurally a tool call, substantively
+        # nothing. Length is a crude proxy, but it separates the two.
+        assert len(str(content)) > 80, (
+            f"{model} submitted a degenerate plan: {content!r}"
+        )
+
+
+@pytest.mark.parametrize("model", MODELS)
+def test_model_round_trips_a_tool_result(model: str) -> None:
+    """The executor loop feeds results back; a model that cannot consume its
+    own tool result cannot run a milestone."""
+    normalized, passthrough = clients()
+    client = select_client(model, normalized, passthrough)
+    events = user("List the files in the current directory using your tools.")
+    produced = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.")
+    calls = [e for e in produced if isinstance(e, ToolCall)]
+    assert calls, f"{model} made no tool call"
+
+    next_id = 2
+    for event in produced:
+        events.append(StoredEvent(id=next_id, ts="t", event=event))
+        next_id += 1
+    events.append(
+        StoredEvent(
+            id=next_id,
+            ts="t",
+            event=ToolResult(
+                call_id=calls[0].call_id, result="app.py\ntests/", success=True
+            ),
+        )
+    )
+    follow_up = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.")
+    assert follow_up, f"{model} produced nothing after its tool result"
+
+
+@pytest.mark.parametrize("model", MODELS)
+def test_model_holds_the_prompt_level_json_contract(model: str) -> None:
+    """A14 contracts structured output at the prompt level because vendors
+    differ on JSON enforcement. This measures whether that holds per model."""
+    normalized, passthrough = clients()
+    client = select_client(model, normalized, passthrough)
+    produced = client.complete(
+        user("Is SQLite sufficient for a single-process agent's task state?"),
+        None,
+        model,
+        system=(
+            "Reply with a single JSON object and nothing else: "
+            '{"answer": "...", "confidence": "high"|"medium"|"low", '
+            '"caveats": "..."}'
+        ),
+    )
+    text = "\n".join(e.text for e in produced if isinstance(e, AssistantText))
+    assert parse_answer(text) is not None, (
+        f"{model} did not produce parseable JSON: {text[:200]!r}"
+    )
+
+
+@pytest.mark.parametrize("model", MODELS)
+def test_route_selection_matches_the_model(model: str) -> None:
+    """A non-Anthropic model must not be sent Anthropic-shaped bodies. This
+    was hardcoded until gpt-6-astra made it observable."""
+    normalized, passthrough = clients()
+    chosen = select_client(model, normalized, passthrough)
+    if model.startswith(("claude-", "anthropic/")):
+        assert chosen is passthrough
+    else:
+        assert chosen is normalized
+    # And the chosen route must actually work for this model.
+    try:
+        chosen.complete(user("say pong"), None, model, system="Be terse.")
+    except LLMError as exc:
+        pytest.fail(f"{model} failed on its selected route: {exc}")