dongarandClaude Opus 5.5
494715efa9
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 55s
Every turn re-sent the whole transcript with nothing checking it against the model's window. On a local model that fails silently: Ollama drops the start of an over-long prompt, which is where the system prompt and the plan are. Measured on the Jetson's llama3.2, cut at 4096 tokens with no error. - The model catalog reads max_input_tokens and max_output_tokens from LiteLLM's model_info, and startup refuses a role on a model with no declared window - BudgetedClient wraps both routes. It fits the canonical events before rendering: older tool results cut to head and tail, then the oldest turns dropped with a note, keeping calls with their results; a request that still cannot fit fails with ContextOverflow and is never sent. The output ceiling is the smaller of config and the model's limit - Tokens are estimated from characters, pessimistically, and the estimate is corrected from reported prompt sizes, only ever upward - The executor takes its scope from the log: the latest plan, which is the approved milestone plan, goes in its system prompt, and it sees only events after it - Tool output keeps its head and, mostly, its tail, so a test runner's failure summary survives truncation - tuning.context.keep_recent_turns and safety_margin are new knobs - scripts/probe_context.py checks a declared window against what the server does; it confirmed llama3.2 cuts at the declared 4096 - The tuning guide covers the budget, declaring a local model's limits, and the probe Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
75 lines
2.4 KiB
Python
75 lines
2.4 KiB
Python
"""Check a model's declared context window against what its server does.
|
|
|
|
uv run python scripts/probe_context.py ALIAS [--config PATH]
|
|
|
|
ALIAS is a LiteLLM model alias. The LiteLLM URL and token come from the agent
|
|
config (NISSE_CONFIG, ./nisse-dev.yaml, or --config), with its secrets from
|
|
the environment or .env. See docs/Tuning-Guide.md, "Declaring a local model's
|
|
limits", and nisse_dev/probe.py for what the verdicts mean.
|
|
"""
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
|
|
from nisse_dev.catalog import ModelCatalog
|
|
from nisse_dev.config import AgentConfig, ConfigError
|
|
from nisse_dev.envfile import load_env_file
|
|
from nisse_dev.probe import probe
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
|
|
parser.add_argument("alias", help="LiteLLM model alias to probe")
|
|
parser.add_argument("--config", help="agent config file to take LiteLLM from")
|
|
parser.add_argument(
|
|
"--max-probe-tokens",
|
|
type=int,
|
|
default=64_000,
|
|
help="refuse to send a prompt larger than this (default: %(default)s)",
|
|
)
|
|
args = parser.parse_args()
|
|
|
|
load_env_file(Path(os.environ.get("NISSE_ENV", ".env")).expanduser())
|
|
try:
|
|
config = (
|
|
AgentConfig.from_file(Path(args.config))
|
|
if args.config
|
|
else AgentConfig.load()
|
|
)
|
|
except ConfigError as exc:
|
|
print(exc, file=sys.stderr)
|
|
return 2
|
|
|
|
catalog = ModelCatalog.discover(config.llm_base_url, config.llm_token)
|
|
declared = catalog.facts(args.alias).max_input_tokens
|
|
if declared is None:
|
|
print(
|
|
f"LiteLLM declares no max_input_tokens for {args.alias}; set one first",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
if declared * 1.5 > args.max_probe_tokens:
|
|
print(
|
|
f"{args.alias} declares {declared} tokens; probing would send about "
|
|
f"{int(declared * 1.5)}. Raise --max-probe-tokens to allow it.",
|
|
file=sys.stderr,
|
|
)
|
|
return 1
|
|
|
|
with httpx.Client(
|
|
base_url=config.llm_base_url.rstrip("/"),
|
|
headers={"Authorization": f"Bearer {config.llm_token}"},
|
|
timeout=600,
|
|
) as http:
|
|
result = probe(http, args.alias, declared)
|
|
print(f"{args.alias}: {result.message}")
|
|
return 0 if result.verdict in ("matches", "refuses", "declared_conservative") else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|