import json import httpx from nisse_core.models.probe import interpret, probe def test_a_cut_at_the_declared_window_matches() -> None: result = interpret(declared=4096, sent=6144, reported=4096, error=None) assert result.verdict == "matches" assert result.server_window == 4096 def test_a_cut_below_the_declaration_says_what_to_set() -> None: # The dangerous case: prompts between the two would be silently cut. result = interpret(declared=8192, sent=12288, reported=4096, error=None) assert result.verdict == "declared_too_high" assert "set max_input_tokens to 4096" in result.message def test_a_cut_above_the_declaration_is_conservative() -> None: result = interpret(declared=4096, sent=6144, reported=5000, error=None) assert result.verdict == "declared_conservative" def test_an_uncut_prompt_means_the_server_allows_more() -> None: result = interpret(declared=4096, sent=6144, reported=6100, error=None) assert result.verdict == "declared_conservative" assert result.server_window is None def test_a_refusal_is_the_safe_behaviour() -> None: result = interpret(declared=4096, sent=6144, reported=None, error="400: too long") assert result.verdict == "refuses" def test_probe_calibrates_then_sends_half_again_the_window() -> None: """Against a fake Ollama: 2 tokens a word, silently cut at 4096.""" sizes: list[int] = [] def handler(request: httpx.Request) -> httpx.Response: words = len(json.loads(request.content)["messages"][0]["content"].split()) sizes.append(words * 2) reported = min(words * 2, 4096) return httpx.Response( 200, json={"model": "llama3.2", "usage": {"prompt_tokens": reported}} ) with httpx.Client( base_url="http://litellm", transport=httpx.MockTransport(handler) ) as http: result = probe(http, "llama3.2", declared=4096) assert result.verdict == "matches" assert sizes[1] >= 4096 * 1.4 # the probe really exceeded the window def test_a_fallback_answer_is_never_credited_to_the_model() -> None: """Measured: LiteLLM answered an oversized prompt to local-coder with claude-sonnet-5, and the probe called the refusing server roomy.""" def handler(request: httpx.Request) -> httpx.Response: words = len(json.loads(request.content)["messages"][0]["content"].split()) model = "local-coder" if words < 1000 else "claude-sonnet-5" return httpx.Response( 200, json={"model": model, "usage": {"prompt_tokens": words * 2}} ) with httpx.Client( base_url="http://litellm", transport=httpx.MockTransport(handler) ) as http: result = probe(http, "local-coder", declared=32768, backing="openai/qwen3-14b") assert result.verdict == "fallback" assert "claude-sonnet-5 answered instead of local-coder" in result.message def test_a_provider_reporting_its_own_model_name_is_not_a_fallback() -> None: def handler(request: httpx.Request) -> httpx.Response: return httpx.Response( 200, json={"model": "qwen3-14b", "usage": {"prompt_tokens": 400}} ) with httpx.Client( base_url="http://litellm", transport=httpx.MockTransport(handler) ) as http: result = probe(http, "local-coder", declared=32768, backing="openai/qwen3-14b") assert result.verdict != "fallback"