Log every model call's time and tokens
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 2m57s
CI / e2e (pull_request) Successful in 4m42s

Each call through BudgetedClient emits llm.call: seconds, input,
output and cached tokens, and the answering model when a fallback
answered; a failed call logs how long it ran before it failed.
Usage now carries the server's cached-prompt count on both routes.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-10-20 23:07:18 +00:00
1 parent bd05acb89d
commit 31c97ccfb5
4 files changed
+136 -13

No files matched your search

+43 -11
View File
@@ -31,6 +31,7 @@ dangerous direction.
"""
import json
import time
from collections.abc import Callable, Sequence
from dataclasses import dataclass, replace
from typing import Any
@@ -297,6 +298,31 @@ class BudgetedClient:
def last_rate_limit_wait_s(self) -> float:
return float(getattr(self._inner, "last_rate_limit_wait_s", 0.0) or 0.0)
def _log_call(self, model: str, started: float, error: str | None = None) -> None:
"""One event per model call: how long it took and what it used, so a
slow turn shows whether reading or writing took the time."""
if self._oplog is None:
return
fields: dict[str, Any] = {
"model": model,
"seconds": round(time.monotonic() - started, 1),
}
if error is not None:
self._oplog.emit("llm.call", cause="failed", error=error, **fields)
return
usage = self.last_usage
answered = self.last_model
if answered and answered != model:
fields["answered_by"] = answered
self._oplog.emit(
"llm.call",
cause="answered",
input_tokens=int(usage.get("input", 0)),
output_tokens=int(usage.get("output", 0)),
cached_tokens=int(usage.get("cached", 0)),
**fields,
)
def budget_for(self, model: str) -> Budget:
window, model_max_output = self._limits_for(model)
ceiling = self._max_output_tokens
@@ -360,17 +386,23 @@ class BudgetedClient:
)
if require_tool_call is None:
require_tool_call = bool(tools) and self._enforces_tool_choice(model)
produced = self._inner.complete(
fitted,
tools,
model,
system=system,
max_tokens=budget.max_output,
# Every caller that offers tools needs one called: the executor
# finishes with finish, the planner with submit_plan or
# ask_human, a delegation with hand_back.
require_tool_call=require_tool_call,
)
started = time.monotonic()
try:
produced = self._inner.complete(
fitted,
tools,
model,
system=system,
max_tokens=budget.max_output,
# Every caller that offers tools needs one called: the executor
# finishes with finish, the planner with submit_plan or
# ask_human, a delegation with hand_back.
require_tool_call=require_tool_call,
)
except Exception as e:
self._log_call(model, started, error=type(e).__name__)
raise
self._log_call(model, started)
reported = int(self.last_usage.get("input", 0))
if reported > 0:
estimated = estimate_tokens(fitted, system, tools)
+5
View File
@@ -244,6 +244,10 @@ class LiteLLMClient:
self.last_usage = {
"input": int(usage.get("prompt_tokens") or 0),
"output": int(usage.get("completion_tokens") or 0),
# Prompt tokens the server reused from its cache, when it says.
"cached": int(
(usage.get("prompt_tokens_details") or {}).get("cached_tokens") or 0
),
}
out: list[CanonicalEvent] = []
if message.get("content"):
@@ -371,6 +375,7 @@ class AnthropicPassthroughClient:
self.last_usage = {
"input": int(usage.get("input_tokens") or 0),
"output": int(usage.get("output_tokens") or 0),
"cached": int(usage.get("cache_read_input_tokens") or 0),
}
out: list[CanonicalEvent] = []
for block in content: