nisse/nisse-dev/tests/test_catalog.py

"""Runtime model discovery (architecture L9).

The point of these tests is that nothing here reads the model's *name*. A
semantic alias like "remote-architecture" is free to point anywhere, so the
route has to come from what LiteLLM says the model is.
"""

import httpx
import pytest

from nisse_dev.catalog import (
    CatalogUnavailable,
    ModelCatalog,
    UnknownModel,
    select_client,
)


def entry(alias: str, backing: str, provider: str | None, tools: bool | None = None):
    """One /model/info row, shaped as the live proxy returns it."""
    info: dict = {"supports_function_calling": tools}
    if provider is not None:
        info["litellm_provider"] = provider
    return {
        "model_name": alias,
        "litellm_params": {"model": backing, "api_base": "http://proxy.test/v1"},
        "model_info": info,
    }


def catalog(*entries) -> ModelCatalog:
    return ModelCatalog.from_payload({"data": list(entries)})


# --- parsing -------------------------------------------------------------


def test_facts_are_read_from_the_payload() -> None:
    cat = catalog(
        entry("claude-haiku-4-5", "anthropic/claude-haiku-4-5", "anthropic", True)
    )
    facts = cat.facts("claude-haiku-4-5")
    assert facts.backing == "anthropic/claude-haiku-4-5"
    assert facts.provider == "anthropic"
    assert facts.supports_tools is True
    assert facts.native_anthropic


def test_provider_falls_back_to_the_backing_prefix() -> None:
    # litellm_provider is absent on some deployments; the backing model's
    # own "vendor/model" prefix says the same thing.
    cat = catalog(entry("mystery", "anthropic/claude-opus-5", None))
    assert cat.facts("mystery").provider == "anthropic"


def test_unnamed_entries_are_skipped() -> None:
    cat = ModelCatalog.from_payload({"data": [{"litellm_params": {"model": "x"}}]})
    assert cat.aliases() == []


def test_missing_tool_support_is_unknown_not_false() -> None:
    # LiteLLM reports null for models it has no capability data on. That is
    # not the same as declaring the model tool-less, and the startup warning
    # depends on the difference.
    cat = catalog(entry("local-fast", "openai/qwen3-8b-awq", "openai", None))
    assert cat.facts("local-fast").supports_tools is None


# --- route selection by provider, never by name --------------------------


def test_a_semantic_alias_routes_by_its_provider() -> None:
    # The whole feature: nothing in "remote-architecture" says Anthropic.
    cat = catalog(entry("remote-architecture", "anthropic/claude-opus-5", "anthropic"))
    normalized, passthrough = object(), object()
    chosen = select_client("remote-architecture", normalized, passthrough, cat)
    assert chosen is passthrough


def test_the_same_alias_reroutes_when_it_is_repointed() -> None:
    # L9: provisionally accept LiteLLM's normalization for a vendor with no
    # native adapter, rather than sending it Anthropic-shaped bodies.
    cat = catalog(entry("remote-architecture", "openai/gpt-6-astra", "openai"))
    normalized, passthrough = object(), object()
    assert (
        select_client("remote-architecture", normalized, passthrough, cat) is normalized
    )


def test_a_name_that_merely_mentions_a_vendor_is_not_that_vendor() -> None:
    cat = catalog(entry("claude-clone-9", "openai/some-clone", "openai"))
    normalized, passthrough = object(), object()
    assert select_client("claude-clone-9", normalized, passthrough, cat) is normalized


def test_local_models_take_the_normalized_route() -> None:
    cat = catalog(
        entry("local-coder", "openai/qwen3-14b", "openai"),
        entry("local-fast", "openai/qwen3-8b-awq", "openai"),
    )
    normalized, passthrough = object(), object()
    for alias in ("local-coder", "local-fast"):
        assert select_client(alias, normalized, passthrough, cat) is normalized


# --- misconfiguration is loud --------------------------------------------


def test_an_unknown_model_raises_rather_than_guessing_a_route() -> None:
    cat = catalog(entry("local-fast", "openai/qwen3-8b-awq", "openai"))
    with pytest.raises(UnknownModel, match="local-fsat"):
        select_client("local-fsat", object(), object(), cat)


def test_the_error_lists_what_is_available() -> None:
    cat = catalog(
        entry("local-fast", "openai/qwen3-8b-awq", "openai"),
        entry("claude-opus-5", "anthropic/claude-opus-5", "anthropic"),
    )
    with pytest.raises(UnknownModel, match="claude-opus-5, local-fast"):
        cat.facts("typo")


def test_validate_names_every_bad_role_at_once() -> None:
    # One boot should surface the whole list, not make the operator
    # rediscover the next typo on the next start.
    cat = catalog(entry("local-fast", "openai/qwen3-8b-awq", "openai"))
    with pytest.raises(UnknownModel) as caught:
        cat.validate(
            {
                "executor": "local-fast",
                "planner": "clause-opus-5",
                "helper": "remote-architecture",
                "expert": "local-fast",
            }
        )
    message = str(caught.value)
    assert "planner='clause-opus-5'" in message
    assert "helper='remote-architecture'" in message
    assert "executor" not in message  # the good ones are not noise


def test_validate_passes_a_whole_good_mapping() -> None:
    cat = catalog(
        entry("local-fast", "openai/qwen3-8b-awq", "openai"),
        entry("claude-opus-5", "anthropic/claude-opus-5", "anthropic"),
    )
    cat.validate({"executor": "local-fast", "planner": "claude-opus-5"})


# --- an outage is not a misconfiguration ---------------------------------


def test_an_unreachable_proxy_is_reported_as_an_outage() -> None:
    def handler(request: httpx.Request) -> httpx.Response:
        raise httpx.ConnectError("no route to host")

    with pytest.raises(CatalogUnavailable, match="could not reach"):
        ModelCatalog.discover(
            "http://litellm.test", "sk-test", transport=httpx.MockTransport(handler)
        )


def test_a_refused_listing_is_reported_as_an_outage() -> None:
    cat = httpx.MockTransport(lambda request: httpx.Response(401, text="bad key"))
    with pytest.raises(CatalogUnavailable, match="401"):
        ModelCatalog.discover("http://litellm.test", "sk-test", transport=cat)


def test_discovery_asks_the_proxy_with_the_token() -> None:
    seen: dict = {}

    def handler(request: httpx.Request) -> httpx.Response:
        seen["url"] = str(request.url)
        seen["auth"] = request.headers.get("Authorization")
        return httpx.Response(
            200, json={"data": [entry("local-fast", "openai/qwen3-8b-awq", "openai")]}
        )

    cat = ModelCatalog.discover(
        "http://litellm.test/", "sk-test", transport=httpx.MockTransport(handler)
    )
    assert seen["url"] == "http://litellm.test/model/info"
    assert seen["auth"] == "Bearer sk-test"
    assert cat.aliases() == ["local-fast"]


# --- what gets pinned ----------------------------------------------------


def test_pinning_records_what_the_alias_resolved_to() -> None:
    # An alias may point somewhere else tomorrow; eval history (L9) keyed on
    # the alias alone would average across whatever it pointed at.
    cat = catalog(entry("remote-architecture", "anthropic/claude-opus-5", "anthropic"))
    pinned = cat.pinned({"planner": "remote-architecture"})
    assert pinned == {
        "planner": {
            "alias": "remote-architecture",
            "backing": "anthropic/claude-opus-5",
            "provider": "anthropic",
        }
    }