nisse/nisse-dev/src/nisse_dev/catalog.py

"""Runtime model discovery (spec A5, A11, A13).

LiteLLM is the VM's single LLM destination, which makes it the authority on
what any given model *is*. Asking it at startup keeps vendor knowledge in one
place instead of duplicating it as name patterns in agent code -- and that
stops being a tidiness argument the moment a role alias no longer names its
vendor. An alias like ``remote-architecture`` is deliberately free to point at
Anthropic today and something else tomorrow; nothing in the name can tell us
which request shape the model behind it expects.

Discovery is per key, not global: ``/model/info`` returns only the models the
presented token is allowed to use. So the same check that catches a typo'd
alias also catches a role pointed at a model this key cannot reach.
"""

from collections.abc import Mapping
from dataclasses import dataclass
from typing import Any

import httpx

from nisse_dev.llm import LLMClient, LLMError

# LiteLLM's provider name for models we have a vendor-native adapter for.
ANTHROPIC_PROVIDER = "anthropic"


class CatalogUnavailable(LLMError):
    """The catalog could not be fetched: the proxy is down, unreachable, or
    refused the token. An outage, not a mistake in the configuration."""


class UnknownModel(LLMError):
    """A model was named that this key's catalog does not contain: a typo, a
    stale alias, or a key without access. Always a misconfiguration."""


class UnknownLimits(LLMError):
    """A model in use has no declared context window. LiteLLM knows the
    limits of commercial models; for a local one they are whatever its server
    allocates, and have to be declared in LiteLLM's ``model_info``. Guessing
    is not safe: a server that silently truncates a prompt (Ollama does)
    drops its start, which is where the system prompt and the plan are."""


@dataclass(frozen=True)
class ModelFacts:
    alias: str  # what we ask for, e.g. "remote-architecture"
    backing: str  # what LiteLLM calls, e.g. "anthropic/claude-opus-5"
    provider: str  # "anthropic", "openai", "ollama", ...
    supports_tools: bool | None  # None when LiteLLM does not say
    # The whole context window, prompt plus response, as LiteLLM declares it
    # (model_info.max_input_tokens); None when it does not say.
    max_input_tokens: int | None = None
    # The most one response may use (model_info.max_output_tokens).
    max_output_tokens: int | None = None
    # Whether its server enforces tool_choice (model_info.supports_tool_choice).
    # Only a declared True counts: a model that says nothing is not asked.
    supports_tool_choice: bool | None = None

    @property
    def native_anthropic(self) -> bool:
        return self.provider == ANTHROPIC_PROVIDER

    def answered(self, reported: str | None) -> bool:
        """Whether a response reporting this model name came from this model.

        A proxy fallback answers under another model's name. LiteLLM reports
        the alias; a provider may report its own name for the backing model,
        sometimes with a date suffix ("claude-haiku-4-5-20251001").
        """
        if not reported:
            return True  # nothing reported, nothing contradicted
        own_name = self.backing.partition("/")[2] or self.backing
        return reported in (self.alias, self.backing) or reported.startswith(own_name)


class ModelCatalog:
    """What one LiteLLM key can see, as of the moment it was asked."""

    def __init__(self, facts: Mapping[str, ModelFacts]) -> None:
        self._facts = dict(facts)

    @classmethod
    def from_payload(cls, payload: Mapping[str, Any]) -> "ModelCatalog":
        facts: dict[str, ModelFacts] = {}
        for entry in payload.get("data") or []:
            alias = entry.get("model_name")
            if not alias:
                continue
            params = entry.get("litellm_params") or {}
            info = entry.get("model_info") or {}
            backing = str(params.get("model") or "")
            # litellm_provider is the direct answer; the backing model's
            # "vendor/model" prefix says the same thing and is present even
            # when the provider field is not.
            provider = info.get("litellm_provider") or backing.partition("/")[0]
            facts[alias] = ModelFacts(
                alias=alias,
                backing=backing,
                provider=str(provider),
                supports_tools=info.get("supports_function_calling"),
                max_input_tokens=_positive_int(info.get("max_input_tokens")),
                max_output_tokens=_positive_int(info.get("max_output_tokens")),
                supports_tool_choice=info.get("supports_tool_choice"),
            )
        return cls(facts)

    @classmethod
    def discover(
        cls,
        base_url: str,
        token: str,
        *,
        timeout_s: float = 30.0,
        transport: httpx.BaseTransport | None = None,
    ) -> "ModelCatalog":
        try:
            with httpx.Client(
                base_url=base_url.rstrip("/"),
                headers={"Authorization": f"Bearer {token}"},
                timeout=timeout_s,
                transport=transport,
            ) as http:
                response = http.get("/model/info")
        except httpx.HTTPError as exc:
            raise CatalogUnavailable(
                f"could not reach the LiteLLM proxy at {base_url} to discover "
                f"models: {exc}"
            ) from exc
        if response.status_code != 200:
            raise CatalogUnavailable(
                f"the LiteLLM proxy refused a model listing with "
                f"{response.status_code}: {response.text[:300]}"
            )
        return cls.from_payload(response.json())

    def aliases(self) -> list[str]:
        return sorted(self._facts)

    def facts(self, alias: str) -> ModelFacts:
        try:
            return self._facts[alias]
        except KeyError:
            raise UnknownModel(
                f"{alias!r} is not a model this LiteLLM key can see; "
                f"available: {', '.join(self.aliases()) or '(none)'}"
            ) from None

    def validate(self, roles: Mapping[str, str]) -> None:
        """Check a whole role mapping at once, before anything uses it.

        Reported together rather than one per boot: an operator fixing a
        config wants the whole list, not to rediscover the next typo on the
        next start (the same reason ``AgentConfig`` reports every missing
        setting at once).
        """
        unknown = {
            role: alias for role, alias in roles.items() if alias not in self._facts
        }
        if unknown:
            named = ", ".join(
                f"{role}={alias!r}" for role, alias in sorted(unknown.items())
            )
            raise UnknownModel(
                f"role(s) point at models this LiteLLM key cannot see: {named}; "
                f"available: {', '.join(self.aliases()) or '(none)'}"
            )

    def require_limits(self, roles: Mapping[str, str]) -> None:
        """Refuse a role mapping that uses a model with no declared context
        window, naming every such model at once."""
        missing = {
            role: alias
            for role, alias in roles.items()
            if self.facts(alias).max_input_tokens is None
        }
        if missing:
            named = ", ".join(
                f"{role}={alias!r}" for role, alias in sorted(missing.items())
            )
            raise UnknownLimits(
                f"no context window is declared for: {named}. Set "
                "model_info.max_input_tokens (and max_output_tokens) for these "
                "in LiteLLM; docs/Tuning-Guide.md says how to find the values"
            )

    def pinned(self, roles: Mapping[str, str]) -> dict[str, dict[str, Any]]:
        """The resolved mapping to record at task start (A11, A16).

        Recording only the alias would let an alias that silently changes
        models muddle the eval history behind it: "remote-architecture: 80%
        good" averaged over however many models it pointed at is not a fact
        about anything.
        """
        return {
            role: {
                "alias": alias,
                "backing": self.facts(alias).backing,
                "provider": self.facts(alias).provider,
            }
            for role, alias in roles.items()
        }


def _positive_int(value: Any) -> int | None:
    if isinstance(value, bool) or not isinstance(value, int | float):
        return None
    return int(value) if value > 0 else None


def select_client(
    model: str,
    normalized: LLMClient,
    passthrough: LLMClient,
    catalog: ModelCatalog,
) -> LLMClient:
    """Choose the route from what the model *is*, not from what it is called.

    A13 spells out the rule: use the vendor-native adapter where one exists,
    and otherwise provisionally accept LiteLLM's normalization for that
    vendor -- validating it against the A16 eval set before trusting it with
    delegations. There is deliberately no name-based fallback here: a second
    heuristic could disagree with the proxy, and an unknown model raises
    rather than quietly taking the normalized route.
    """
    return passthrough if catalog.facts(model).native_anthropic else normalized