Discover a model's vendor from LiteLLM instead of its name

Route selection sniffed model names: anything starting "claude-" or
"anthropic/" took the vendor-native passthrough, everything else took the
normalized route. That works only while an alias names its vendor. A
semantic role alias like "remote-architecture" is meant to point at
Anthropic today and somewhere else tomorrow, and the name says nothing
either way -- so it would have taken the normalized route regardless of
what was behind it, silently losing the fidelity A13 wants for delegations.

LiteLLM already knows the answer. /model/info reports each alias's backing
model, provider, and declared tool support, so the agent asks at startup and
routes on the provider it discovers. That keeps vendor knowledge where A5
already put it -- in the proxy -- rather than duplicating it as name patterns
here.

There is deliberately no name-based fallback: PASSTHROUGH_PREFIXES and
has_native_adapter are gone, and select_client now requires a catalog. A
second heuristic could only ever disagree with the proxy. An unknown model
raises instead of guessing a route.

Misconfiguration is therefore loud and early. build_agent validates every
role against the catalog before it clones anything, naming all the bad ones
at once. Discovery is per key -- /model/info returns only what the presented
token may use -- so the same check catches a role pointed at a real model
this key cannot reach. An unreachable proxy is reported separately, since an
outage and a typo call for different responses from the operator.

Two things now get recorded rather than inferred: the resolved model is
pinned alongside the role mapping at task start, for the reason A11 already
pins roles, and DelegationStarted carries backing_model so A16 eval history
is not averaged across whatever an alias happened to point at.

A model LiteLLM declares tool-less warns at startup rather than refusing;
A15 expects weak local models to be accommodated, and they are observed
driving tool loops anyway.

Verified live: all four aliases the production key can see route correctly
through discovery, and each chosen route works.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

nisse-dev/src/nisse_dev/agent.py

@@ -12,6 +12,7 @@ from collections.abc import Callable
 from dataclasses import dataclass
 
 from nisse_dev.approvals import PlanBoard
+from nisse_dev.catalog import ModelCatalog, select_client
 from nisse_dev.config import AgentConfig
 from nisse_dev.control import Orchestrator
 from nisse_dev.delegation import Delegation
@@ -19,11 +20,7 @@ from nisse_dev.executor import LLMExecutor
 from nisse_dev.expert import ExpertDesk
 from nisse_dev.git import GitRepo, authenticated_url
 from nisse_dev.gitea import Comment, GiteaClient
-from nisse_dev.llm import (
-    AnthropicPassthroughClient,
-    LiteLLMClient,
-    select_client,
-)
+from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
 from nisse_dev.matrix import MatrixTransport
 from nisse_dev.oplog import OpLog
 from nisse_dev.planner import LLMPlanningAgent
@@ -32,6 +29,35 @@ from nisse_dev.publish import MilestonePublisher
 from nisse_dev.state import StateStore
 from nisse_dev.tools import Workspace
 
+TOOL_DRIVING_ROLES = ("planner", "executor", "delegate")
+
+
+def discover_models(config: AgentConfig, oplog: OpLog) -> ModelCatalog:
+    """Resolve every configured role against LiteLLM before anything runs.
+
+    Refusing to boot is the whole point: a role pointed at a model this key
+    cannot see is a misconfiguration, and the cheapest place to learn that is
+    here rather than ten minutes into a milestone. An unreachable proxy is
+    reported separately -- it is an outage, not a mistake, and telling them
+    apart is what lets an operator know whether to edit config or wait.
+    """
+    catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token)
+    catalog.validate(config.roles.pinned())
+    for role in TOOL_DRIVING_ROLES:
+        facts = catalog.facts(config.roles.resolve(role))
+        if facts.supports_tools is False:
+            # A warning, not a refusal: models LiteLLM declares tool-less are
+            # observed driving tool loops anyway, and A15 expects weak local
+            # models to be accommodated rather than excluded.
+            oplog.emit(
+                "model_tools_unsupported",
+                cause="startup",
+                role=role,
+                model=facts.alias,
+                backing=facts.backing,
+            )
+    return catalog
+
 
 @dataclass
 class Agent:
@@ -91,6 +117,9 @@ def build_agent(
 ) -> Agent:
     oplog = OpLog("nisse-dev")
     store = StateStore(config.state_db, oplog=oplog)
+    # Before the clone, before the token checks: a typo'd role name should
+    # cost a second, not a repository.
+    catalog = discover_models(config, oplog)
 
     gitea = GiteaClient(config.gitea_url, config.gitea_token)
     # Whoami rather than another config variable: it names the identity every
@@ -113,11 +142,12 @@ def build_agent(
     expert = ExpertDesk(normalized, store, config.roles)
     delegate_model = config.roles.resolve("delegate")
     delegation = Delegation(
-        select_client(delegate_model, normalized, passthrough),
+        select_client(delegate_model, normalized, passthrough, catalog),
         store,
         workspace,
         git_repo,
         model=delegate_model,
+        backing_model=catalog.facts(delegate_model).backing,
     )
     executor = LLMExecutor(
         normalized,
@@ -162,6 +192,7 @@ def build_agent(
         board=PlanBoard(gitea, config.gitea_repo),
         delegate=delegation,
         poller_factory=poller_factory,
+        catalog=catalog,
     )
     return Agent(
         config=config,

nisse-dev/src/nisse_dev/canonical.py

@@ -56,8 +56,12 @@ class GateOpened:
 class DelegationStarted:
     brief: str
     role: str
-    model: str
+    model: str  # the alias asked for, e.g. "remote-architecture"
     caps: dict[str, Any]  # turns / tokens / wall-clock limits
+    # What the alias resolved to at task start. An alias is free to change
+    # models between tasks, so eval history (A16) keyed on the alias alone
+    # would average results across whatever it happened to point at.
+    backing_model: str = ""
 
 
 @dataclass(frozen=True)

nisse-dev/src/nisse_dev/catalog.py

@@ -0,0 +1,172 @@
+"""Runtime model discovery (spec A5, A11, A13).
+
+LiteLLM is the VM's single LLM destination, which makes it the authority on
+what any given model *is*. Asking it at startup keeps vendor knowledge in one
+place instead of duplicating it as name patterns in agent code -- and that
+stops being a tidiness argument the moment a role alias no longer names its
+vendor. An alias like ``remote-architecture`` is deliberately free to point at
+Anthropic today and something else tomorrow; nothing in the name can tell us
+which request shape the model behind it expects.
+
+Discovery is per key, not global: ``/model/info`` returns only the models the
+presented token is allowed to use. So the same check that catches a typo'd
+alias also catches a role pointed at a model this key cannot reach.
+"""
+
+from collections.abc import Mapping
+from dataclasses import dataclass
+from typing import Any
+
+import httpx
+
+from nisse_dev.llm import LLMClient, LLMError
+
+# LiteLLM's provider name for models we have a vendor-native adapter for.
+ANTHROPIC_PROVIDER = "anthropic"
+
+
+class CatalogUnavailable(LLMError):
+    """The catalog could not be fetched: the proxy is down, unreachable, or
+    refused the token. An outage, not a mistake in the configuration."""
+
+
+class UnknownModel(LLMError):
+    """A model was named that this key's catalog does not contain: a typo, a
+    stale alias, or a key without access. Always a misconfiguration."""
+
+
+@dataclass(frozen=True)
+class ModelFacts:
+    alias: str  # what we ask for, e.g. "remote-architecture"
+    backing: str  # what LiteLLM calls, e.g. "anthropic/claude-opus-5"
+    provider: str  # "anthropic", "openai", "ollama", ...
+    supports_tools: bool | None  # None when LiteLLM does not say
+
+    @property
+    def native_anthropic(self) -> bool:
+        return self.provider == ANTHROPIC_PROVIDER
+
+
+class ModelCatalog:
+    """What one LiteLLM key can see, as of the moment it was asked."""
+
+    def __init__(self, facts: Mapping[str, ModelFacts]) -> None:
+        self._facts = dict(facts)
+
+    @classmethod
+    def from_payload(cls, payload: Mapping[str, Any]) -> "ModelCatalog":
+        facts: dict[str, ModelFacts] = {}
+        for entry in payload.get("data") or []:
+            alias = entry.get("model_name")
+            if not alias:
+                continue
+            params = entry.get("litellm_params") or {}
+            info = entry.get("model_info") or {}
+            backing = str(params.get("model") or "")
+            # litellm_provider is the direct answer; the backing model's
+            # "vendor/model" prefix says the same thing and is present even
+            # when the provider field is not.
+            provider = info.get("litellm_provider") or backing.partition("/")[0]
+            facts[alias] = ModelFacts(
+                alias=alias,
+                backing=backing,
+                provider=str(provider),
+                supports_tools=info.get("supports_function_calling"),
+            )
+        return cls(facts)
+
+    @classmethod
+    def discover(
+        cls,
+        base_url: str,
+        token: str,
+        *,
+        timeout_s: float = 30.0,
+        transport: httpx.BaseTransport | None = None,
+    ) -> "ModelCatalog":
+        try:
+            with httpx.Client(
+                base_url=base_url.rstrip("/"),
+                headers={"Authorization": f"Bearer {token}"},
+                timeout=timeout_s,
+                transport=transport,
+            ) as http:
+                response = http.get("/model/info")
+        except httpx.HTTPError as exc:
+            raise CatalogUnavailable(
+                f"could not reach the LiteLLM proxy at {base_url} to discover "
+                f"models: {exc}"
+            ) from exc
+        if response.status_code != 200:
+            raise CatalogUnavailable(
+                f"the LiteLLM proxy refused a model listing with "
+                f"{response.status_code}: {response.text[:300]}"
+            )
+        return cls.from_payload(response.json())
+
+    def aliases(self) -> list[str]:
+        return sorted(self._facts)
+
+    def facts(self, alias: str) -> ModelFacts:
+        try:
+            return self._facts[alias]
+        except KeyError:
+            raise UnknownModel(
+                f"{alias!r} is not a model this LiteLLM key can see; "
+                f"available: {', '.join(self.aliases()) or '(none)'}"
+            ) from None
+
+    def validate(self, roles: Mapping[str, str]) -> None:
+        """Check a whole role mapping at once, before anything uses it.
+
+        Reported together rather than one per boot: an operator fixing a
+        config wants the whole list, not to rediscover the next typo on the
+        next start (the same reason ``AgentConfig.from_env`` batches its
+        missing variables).
+        """
+        unknown = {
+            role: alias for role, alias in roles.items() if alias not in self._facts
+        }
+        if unknown:
+            named = ", ".join(
+                f"{role}={alias!r}" for role, alias in sorted(unknown.items())
+            )
+            raise UnknownModel(
+                f"role(s) point at models this LiteLLM key cannot see: {named}; "
+                f"available: {', '.join(self.aliases()) or '(none)'}"
+            )
+
+    def pinned(self, roles: Mapping[str, str]) -> dict[str, dict[str, Any]]:
+        """The resolved mapping to record at task start (A11, A16).
+
+        Recording only the alias would let an alias that silently changes
+        models muddle the eval history behind it: "remote-architecture: 80%
+        good" averaged over however many models it pointed at is not a fact
+        about anything.
+        """
+        return {
+            role: {
+                "alias": alias,
+                "backing": self.facts(alias).backing,
+                "provider": self.facts(alias).provider,
+            }
+            for role, alias in roles.items()
+        }
+
+
+def select_client(
+    model: str,
+    normalized: LLMClient,
+    passthrough: LLMClient,
+    catalog: ModelCatalog,
+) -> LLMClient:
+    """Choose the route from what the model *is*, not from what it is called.
+
+    A13 spells out the rule: use the vendor-native adapter where one exists,
+    and otherwise provisionally accept LiteLLM's normalization for that
+    vendor -- validating it against the A16 eval set before trusting it with
+    delegations. There is deliberately no name-based fallback here: a second
+    heuristic could disagree with the proxy, and an unknown model raises
+    rather than quietly taking the normalized route.
+    """
+    return passthrough if catalog.facts(model).native_anthropic else normalized

nisse-dev/src/nisse_dev/control.py

@@ -143,6 +143,10 @@ def _fresh_data() -> dict[str, Any]:
         "milestone": 1,
         "task_id": None,
         "roles": None,  # A11 mapping, pinned at task start
+        # What each pinned alias resolved to, discovered from LiteLLM. An
+        # alias is free to point somewhere else tomorrow, so the transcript
+        # has to say what actually answered (A11, A16).
+        "models": None,
         "pr": None,  # the open PR for the current milestone, once published
         # Pointer to the Gitea plan record (Part H): the state store holds
         # only this, so gate state can be re-derived from Gitea after a
@@ -165,11 +169,13 @@ class Orchestrator:
         board: "PlanBoard | None" = None,
         delegate: Any = None,
         poller_factory: Any = None,
+        catalog: Any = None,
     ) -> None:
         self._store = store
         self._transport = transport
         self._agent = agent
         self._roles = roles
+        self._catalog = catalog
         self._executor = executor
         self._publisher = publisher
         self._operators = set(operators)
@@ -287,7 +293,17 @@ class Orchestrator:
         # Pin the role mapping now (A11): a mid-task config change must not
         # split one task across two planners.
         pinned = self._roles.pinned() if self._roles is not None else None
-        self._data = {**_fresh_data(), "task_id": task_id, "roles": pinned}
+        resolved = (
+            self._catalog.pinned(pinned)
+            if self._catalog is not None and pinned is not None
+            else None
+        )
+        self._data = {
+            **_fresh_data(),
+            "task_id": task_id,
+            "roles": pinned,
+            "models": resolved,
+        }
         self._project = project
         self._store.oplog.task_id = task_id
         if self._board is not None:

nisse-dev/src/nisse_dev/delegation.py

@@ -165,6 +165,7 @@ class Delegation:
         git_repo: GitRepo,
         *,
         model: str,
+        backing_model: str = "",
         caps: Caps | None = None,
         now: Callable[[], float] = time.monotonic,
     ) -> None:
@@ -173,6 +174,7 @@ class Delegation:
         self._workspace = workspace
         self._git = git_repo
         self._model = model
+        self._backing_model = backing_model
         self._caps = caps or Caps()
         self._now = now
 
@@ -210,6 +212,7 @@ class Delegation:
                 role=role,
                 model=self._model,
                 caps=self._caps.as_payload(),
+                backing_model=self._backing_model,
             )
         )
         result = self._loop(brief, branch, started_at)

nisse-dev/src/nisse_dev/llm.py

@@ -371,28 +371,6 @@ def _post_with_rate_limit_retry(
 # Model families we have a native renderer adapter for. A13: delegations take
 # the vendor's passthrough route because a normalising middlebox is lossy
 # exactly where a long-running delegation lives.
-PASSTHROUGH_PREFIXES = ("claude-", "anthropic/")
-
-
-def has_native_adapter(model: str) -> bool:
-    return model.startswith(PASSTHROUGH_PREFIXES)
-
-
-def select_client(
-    model: str, normalized: LLMClient, passthrough: LLMClient
-) -> LLMClient:
-    """Choose the route from the model, not from where the caller sits.
-
-    A13 spells out the rule: use the vendor-native adapter where one exists,
-    and otherwise *provisionally* accept LiteLLM's normalization for that
-    vendor — validating it against the A16 eval set before trusting it with
-    delegations. Hard-wiring the passthrough would mean pointing a role at a
-    non-Anthropic model silently sent it Anthropic-shaped request bodies,
-    which is the opposite of the vendor fluidity A5 requires.
-    """
-    return passthrough if has_native_adapter(model) else normalized
-
-
 def _raise_for_upstream(response: httpx.Response) -> None:
     """Classify a non-200 as upstream trouble where it plainly is.