Log every model call's time and tokens
Each call through BudgetedClient emits llm.call: seconds, input, output and cached tokens, and the answering model when a fallback answered; a failed call logs how long it ran before it failed. Usage now carries the server's cached-prompt count on both routes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
1 parent
bd05acb89d
commit
31c97ccfb5
4 files changed
+136
-13
No files matched your search
@@ -484,6 +484,48 @@ def test_the_answering_model_is_recorded() -> None:
|
||||
assert client.last_model == "gpt-test-1"
|
||||
|
||||
|
||||
def test_usage_includes_what_the_server_reused_from_its_cache() -> None:
|
||||
client = make_client(
|
||||
lambda request: httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "qwen",
|
||||
"choices": [{"message": {"role": "assistant", "content": "hi"}}],
|
||||
"usage": {
|
||||
"prompt_tokens": 900,
|
||||
"completion_tokens": 40,
|
||||
"prompt_tokens_details": {"cached_tokens": 800},
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
client.complete(EVENTS, tools=None, model="qwen")
|
||||
assert client.last_usage == {"input": 900, "output": 40, "cached": 800}
|
||||
|
||||
passthrough = AnthropicPassthroughClient(
|
||||
"https://litellm.test",
|
||||
"tok",
|
||||
transport=httpx.MockTransport(
|
||||
lambda request: httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "claude-opus-5-5",
|
||||
"stop_reason": "end_turn",
|
||||
"content": [{"type": "text", "text": "hi"}],
|
||||
"usage": {
|
||||
"input_tokens": 50,
|
||||
"output_tokens": 7,
|
||||
"cache_read_input_tokens": 30,
|
||||
},
|
||||
},
|
||||
)
|
||||
),
|
||||
sleep=lambda _s: None,
|
||||
)
|
||||
passthrough.complete(EVENTS, tools=None, model="claude-opus-5-5")
|
||||
assert passthrough.last_usage == {"input": 50, "output": 7, "cached": 30}
|
||||
|
||||
|
||||
# --- outages delay, never fail (L5) --------------------------------------------
|
||||
|
||||
|
||||
|
||||
Reference in new issue
Block a user