dongarandClaude Opus 5.5
154a0c3fa8
What isn't the dev agent's moves to nisse-core, unchanged in behaviour, with its tests: canonical, render, render_anthropic, llm, catalog, probe, context, views, state, reconcile, worker, cancel, dump, transport, matrix, execution, testing, vault, memory, oplog, envfile and gitea (whole). - nisse_core.state.StateStore holds the tables every agent has; an agent's subclass adds its tables (version, migrations), roles and views. nisse_dev.state.StateStore adds the build VM blocks (v5). - views.project takes what each role is shown, rather than a fixed table. - Vault takes the agent's name (agent=), for provenance. - A test that nisse_core never imports an agent. - The loop, config, git, delegation, expert, tools and evals stay in nisse-dev until nisse-ops has a loop to share. Docs: the Development Plan marks M7.1 done and notes that Phase 6 waits on the agent host and Terraform; Architecture Part L names the package. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
90 lines
3.3 KiB
Python
90 lines
3.3 KiB
Python
import json
|
|
|
|
import httpx
|
|
|
|
from nisse_core.probe import interpret, probe
|
|
|
|
|
|
def test_a_cut_at_the_declared_window_matches() -> None:
|
|
result = interpret(declared=4096, sent=6144, reported=4096, error=None)
|
|
assert result.verdict == "matches"
|
|
assert result.server_window == 4096
|
|
|
|
|
|
def test_a_cut_below_the_declaration_says_what_to_set() -> None:
|
|
# The dangerous case: prompts between the two would be silently cut.
|
|
result = interpret(declared=8192, sent=12288, reported=4096, error=None)
|
|
assert result.verdict == "declared_too_high"
|
|
assert "set max_input_tokens to 4096" in result.message
|
|
|
|
|
|
def test_a_cut_above_the_declaration_is_conservative() -> None:
|
|
result = interpret(declared=4096, sent=6144, reported=5000, error=None)
|
|
assert result.verdict == "declared_conservative"
|
|
|
|
|
|
def test_an_uncut_prompt_means_the_server_allows_more() -> None:
|
|
result = interpret(declared=4096, sent=6144, reported=6100, error=None)
|
|
assert result.verdict == "declared_conservative"
|
|
assert result.server_window is None
|
|
|
|
|
|
def test_a_refusal_is_the_safe_behaviour() -> None:
|
|
result = interpret(declared=4096, sent=6144, reported=None, error="400: too long")
|
|
assert result.verdict == "refuses"
|
|
|
|
|
|
def test_probe_calibrates_then_sends_half_again_the_window() -> None:
|
|
"""Against a fake Ollama: 2 tokens a word, silently cut at 4096."""
|
|
sizes: list[int] = []
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
words = len(json.loads(request.content)["messages"][0]["content"].split())
|
|
sizes.append(words * 2)
|
|
reported = min(words * 2, 4096)
|
|
return httpx.Response(
|
|
200, json={"model": "llama3.2", "usage": {"prompt_tokens": reported}}
|
|
)
|
|
|
|
with httpx.Client(
|
|
base_url="http://litellm", transport=httpx.MockTransport(handler)
|
|
) as http:
|
|
result = probe(http, "llama3.2", declared=4096)
|
|
|
|
assert result.verdict == "matches"
|
|
assert sizes[1] >= 4096 * 1.4 # the probe really exceeded the window
|
|
|
|
|
|
def test_a_fallback_answer_is_never_credited_to_the_model() -> None:
|
|
"""Measured: LiteLLM answered an oversized prompt to local-coder with
|
|
claude-sonnet-5, and the probe called the refusing server roomy."""
|
|
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
words = len(json.loads(request.content)["messages"][0]["content"].split())
|
|
model = "local-coder" if words < 1000 else "claude-sonnet-5"
|
|
return httpx.Response(
|
|
200, json={"model": model, "usage": {"prompt_tokens": words * 2}}
|
|
)
|
|
|
|
with httpx.Client(
|
|
base_url="http://litellm", transport=httpx.MockTransport(handler)
|
|
) as http:
|
|
result = probe(http, "local-coder", declared=32768, backing="openai/qwen3-14b")
|
|
|
|
assert result.verdict == "fallback"
|
|
assert "claude-sonnet-5 answered instead of local-coder" in result.message
|
|
|
|
|
|
def test_a_provider_reporting_its_own_model_name_is_not_a_fallback() -> None:
|
|
def handler(request: httpx.Request) -> httpx.Response:
|
|
return httpx.Response(
|
|
200, json={"model": "qwen3-14b", "usage": {"prompt_tokens": 400}}
|
|
)
|
|
|
|
with httpx.Client(
|
|
base_url="http://litellm", transport=httpx.MockTransport(handler)
|
|
) as http:
|
|
result = probe(http, "local-coder", declared=32768, backing="openai/qwen3-14b")
|
|
|
|
assert result.verdict != "fallback"
|