Choose the LLM route by model, and add a capability matrix

The arrival of non-Anthropic models made a latent bug observable: the
delegate's client was hardcoded to the Anthropic passthrough, so
pointing NISSE_ROLE_DELEGATE at gpt-6-astra or a local model would
have sent Anthropic-shaped /v1/messages bodies to something that does
not speak them. A5 promises the frontier tier is swappable "as a
configuration change, not a refactor", and it was not.

select_client now picks the route from the model: the vendor-native
adapter where one exists, LiteLLM's normalization otherwise — which is
exactly what A13 prescribes for a vendor whose renderer adapter has
not been written, pending validation against the A16 eval set.

Also adds the capability matrix Part H asks for. It says the
thresholds that depend on local capability — A12's failure count,
A14's consult budget, A15's manifest size — must be tuned "on the real
hardware", and that needs measurement rather than reputation. The
harness probes each model for the things that actually break: whether
it calls a tool at all, whether the plan it submits has substance
(llama3.2 emitted "{"), whether it can consume its own tool result,
and whether prompt-level JSON holds. Opt in with
NISSE_CAPABILITY_MODELS; it spends real tokens across several models.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

nisse-dev/src/nisse_dev/agent.py

@@ -19,7 +19,11 @@ from nisse_dev.executor import LLMExecutor
 from nisse_dev.expert import ExpertDesk
 from nisse_dev.git import GitRepo, authenticated_url
 from nisse_dev.gitea import Comment, GiteaClient
-from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
+from nisse_dev.llm import (
+    AnthropicPassthroughClient,
+    LiteLLMClient,
+    select_client,
+)
 from nisse_dev.matrix import MatrixTransport
 from nisse_dev.oplog import OpLog
 from nisse_dev.planner import LLMPlanningAgent
@@ -98,19 +102,22 @@ def build_agent(
     workspace = Workspace(config.workspace)
 
     normalized = LiteLLMClient(config.llm_base_url, config.llm_token)
-    # Delegations take the passthrough route; fidelity is cheap elsewhere (A5).
+    # Delegations prefer the vendor-native route; fidelity is cheap
+    # elsewhere (A5). Which route a given model gets is decided by
+    # select_client, not by where the caller happens to sit.
     passthrough = AnthropicPassthroughClient(config.llm_base_url, config.llm_token)
 
     transport = MatrixTransport(config.matrix)
     announce = notify or transport.send_message
 
     expert = ExpertDesk(normalized, store, config.roles)
+    delegate_model = config.roles.resolve("delegate")
     delegation = Delegation(
-        passthrough,
+        select_client(delegate_model, normalized, passthrough),
         store,
         workspace,
         git_repo,
-        model=config.roles.resolve("delegate"),
+        model=delegate_model,
     )
     executor = LLMExecutor(
         normalized,

nisse-dev/src/nisse_dev/llm.py

@@ -276,3 +276,28 @@ def _post_with_rate_limit_retry(
             return response
         sleep(RATE_LIMIT_RETRY_S)
     raise AssertionError("unreachable")
+
+
+# Model families we have a native renderer adapter for. A13: delegations take
+# the vendor's passthrough route because a normalising middlebox is lossy
+# exactly where a long-running delegation lives.
+PASSTHROUGH_PREFIXES = ("claude-", "anthropic/")
+
+
+def has_native_adapter(model: str) -> bool:
+    return model.startswith(PASSTHROUGH_PREFIXES)
+
+
+def select_client(
+    model: str, normalized: LLMClient, passthrough: LLMClient
+) -> LLMClient:
+    """Choose the route from the model, not from where the caller sits.
+
+    A13 spells out the rule: use the vendor-native adapter where one exists,
+    and otherwise *provisionally* accept LiteLLM's normalization for that
+    vendor — validating it against the A16 eval set before trusting it with
+    delegations. Hard-wiring the passthrough would mean pointing a role at a
+    non-Anthropic model silently sent it Anthropic-shaped request bodies,
+    which is the opposite of the vendor fluidity A5 requires.
+    """
+    return passthrough if has_native_adapter(model) else normalized

nisse-dev/tests/test_llm.py

@@ -11,7 +11,14 @@ from nisse_dev.canonical import (
     UserMessage,
 )
 from nisse_dev.envfile import load_env_file
-from nisse_dev.llm import RATE_LIMIT_RETRY_S, LiteLLMClient, LLMError, RoleMap
+from nisse_dev.llm import (
+    RATE_LIMIT_RETRY_S,
+    LiteLLMClient,
+    LLMError,
+    RoleMap,
+    has_native_adapter,
+    select_client,
+)
 
 EVENTS = [StoredEvent(id=1, ts="t", event=UserMessage(text="hi", source="m:1"))]
 
@@ -203,3 +210,28 @@ def test_load_env_file_missing_is_noop(tmp_path: Path) -> None:
     env: dict[str, str] = {}
     load_env_file(tmp_path / "nope.env", env)
     assert env == {}
+
+
+# --- route selection (A5 vendor fluidity) --------------------------------
+
+
+def test_anthropic_models_take_the_native_route() -> None:
+    normalized, passthrough = object(), object()
+    for model in ("claude-haiku-4-5", "claude-opus-5", "anthropic/claude-sonnet-5"):
+        assert select_client(model, normalized, passthrough) is passthrough
+
+
+def test_other_vendors_take_the_normalized_route() -> None:
+    # A13: provisionally accept LiteLLM's normalization for a vendor whose
+    # renderer adapter does not exist yet, rather than sending it request
+    # bodies shaped for someone else.
+    normalized, passthrough = object(), object()
+    for model in ("gpt-6-astra", "local-coder", "local-fast", "llama3.2"):
+        assert select_client(model, normalized, passthrough) is normalized
+
+
+def test_native_adapter_detection_is_explicit() -> None:
+    assert has_native_adapter("claude-opus-5")
+    assert not has_native_adapter("gpt-6-astra")
+    # A model merely mentioning the vendor is not the vendor.
+    assert not has_native_adapter("my-claude-clone")