Log every model call's time and tokens
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 2m57s
CI / e2e (pull_request) Successful in 4m42s

Each call through BudgetedClient emits llm.call: seconds, input,
output and cached tokens, and the answering model when a fallback
answered; a failed call logs how long it ran before it failed.
Usage now carries the server's cached-prompt count on both routes.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-10-20 23:07:18 +00:00
1 parent bd05acb89d
commit 31c97ccfb5
4 files changed
+136 -13

No files matched your search

+46 -2
View File
@@ -1,6 +1,7 @@
"""The context budget (architecture L6)."""
import io
import json
from pathlib import Path
from typing import Any
@@ -25,6 +26,7 @@ from nisse_core.models.context import (
fit,
head_and_tail,
)
from nisse_core.models.llm import LLMError
from nisse_core.oplog import OpLog
from nisse_core.state.store import StateStore
@@ -190,7 +192,7 @@ def test_reported_usage_only_ever_raises_the_estimate() -> None:
assert under_then_over._factors["m"] == 2.0 # never loosened
def test_a_request_that_fits_is_sent_whole_and_nothing_is_logged() -> None:
def test_a_request_that_fits_is_sent_whole_and_no_cut_is_logged() -> None:
stream = io.StringIO()
client = BudgetedClient(
RecordingClient(), lambda m: (100_000, None), oplog=OpLog("t", stream)
@@ -198,7 +200,49 @@ def test_a_request_that_fits_is_sent_whole_and_nothing_is_logged() -> None:
events = transcript(3, 100)
client.complete(events, None, "m")
assert client.last_trim is None
assert stream.getvalue() == ""
assert "context.trimmed" not in stream.getvalue()
def logged(stream: io.StringIO, event: str) -> list[dict[str, Any]]:
records = [json.loads(line) for line in stream.getvalue().splitlines()]
return [r for r in records if r["event"] == event]
def test_every_call_logs_its_time_and_tokens() -> None:
# A slow turn should show whether reading or writing took the time (#174).
stream = io.StringIO()
inner = RecordingClient(reported_input=900)
inner.last_usage["cached"] = 800
inner.last_model = "fallback-model"
client = BudgetedClient(inner, lambda m: (100_000, None), oplog=OpLog("t", stream))
client.complete(transcript(1, 10), None, "m")
[call] = logged(stream, "llm.call")
assert call["cause"] == "answered"
assert call["model"] == "m"
assert call["answered_by"] == "fallback-model"
assert (call["input_tokens"], call["output_tokens"], call["cached_tokens"]) == (
900,
5,
800,
)
assert isinstance(call["seconds"], float)
def test_a_failed_call_logs_how_long_it_ran_and_still_raises() -> None:
class TimesOut(RecordingClient):
def complete(self, *args, **kwargs):
raise LLMError("LLM request returned 408: timed out")
stream = io.StringIO()
client = BudgetedClient(
TimesOut(), lambda m: (100_000, None), oplog=OpLog("t", stream)
)
with pytest.raises(LLMError):
client.complete(transcript(1, 10), None, "m")
[call] = logged(stream, "llm.call")
assert call["cause"] == "failed"
assert call["error"] == "LLMError"
assert "input_tokens" not in call
def test_a_cut_is_recorded_and_logged() -> None:
+42
View File
@@ -484,6 +484,48 @@ def test_the_answering_model_is_recorded() -> None:
assert client.last_model == "gpt-test-1"
def test_usage_includes_what_the_server_reused_from_its_cache() -> None:
client = make_client(
lambda request: httpx.Response(
200,
json={
"model": "qwen",
"choices": [{"message": {"role": "assistant", "content": "hi"}}],
"usage": {
"prompt_tokens": 900,
"completion_tokens": 40,
"prompt_tokens_details": {"cached_tokens": 800},
},
},
)
)
client.complete(EVENTS, tools=None, model="qwen")
assert client.last_usage == {"input": 900, "output": 40, "cached": 800}
passthrough = AnthropicPassthroughClient(
"https://litellm.test",
"tok",
transport=httpx.MockTransport(
lambda request: httpx.Response(
200,
json={
"model": "claude-opus-5-5",
"stop_reason": "end_turn",
"content": [{"type": "text", "text": "hi"}],
"usage": {
"input_tokens": 50,
"output_tokens": 7,
"cache_read_input_tokens": 30,
},
},
)
),
sleep=lambda _s: None,
)
passthrough.complete(EVENTS, tools=None, model="claude-opus-5-5")
assert passthrough.last_usage == {"input": 50, "output": 7, "cached": 30}
# --- outages delay, never fail (L5) --------------------------------------------