Log every model call's time and tokens
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 2m57s
CI / e2e (pull_request) Successful in 4m42s

Each call through BudgetedClient emits llm.call: seconds, input,
output and cached tokens, and the answering model when a fallback
answered; a failed call logs how long it ran before it failed.
Usage now carries the server's cached-prompt count on both routes.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-10-20 23:07:18 +00:00
1 parent bd05acb89d
commit 31c97ccfb5
4 files changed
+136 -13

No files matched your search

+42
View File
@@ -484,6 +484,48 @@ def test_the_answering_model_is_recorded() -> None:
assert client.last_model == "gpt-test-1"
def test_usage_includes_what_the_server_reused_from_its_cache() -> None:
client = make_client(
lambda request: httpx.Response(
200,
json={
"model": "qwen",
"choices": [{"message": {"role": "assistant", "content": "hi"}}],
"usage": {
"prompt_tokens": 900,
"completion_tokens": 40,
"prompt_tokens_details": {"cached_tokens": 800},
},
},
)
)
client.complete(EVENTS, tools=None, model="qwen")
assert client.last_usage == {"input": 900, "output": 40, "cached": 800}
passthrough = AnthropicPassthroughClient(
"https://litellm.test",
"tok",
transport=httpx.MockTransport(
lambda request: httpx.Response(
200,
json={
"model": "claude-opus-5-5",
"stop_reason": "end_turn",
"content": [{"type": "text", "text": "hi"}],
"usage": {
"input_tokens": 50,
"output_tokens": 7,
"cache_read_input_tokens": 30,
},
},
)
),
sleep=lambda _s: None,
)
passthrough.complete(EVENTS, tools=None, model="claude-opus-5-5")
assert passthrough.last_usage == {"input": 50, "output": 7, "cached": 30}
# --- outages delay, never fail (L5) --------------------------------------------