Log every model call's time and tokens
Each call through BudgetedClient emits llm.call: seconds, input, output and cached tokens, and the answering model when a fallback answered; a failed call logs how long it ran before it failed. Usage now carries the server's cached-prompt count on both routes. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
1 parent
bd05acb89d
commit
31c97ccfb5
4 files changed
+136
-13
No files matched your search
@@ -31,6 +31,7 @@ dangerous direction.
|
||||
"""
|
||||
|
||||
import json
|
||||
import time
|
||||
from collections.abc import Callable, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from typing import Any
|
||||
@@ -297,6 +298,31 @@ class BudgetedClient:
|
||||
def last_rate_limit_wait_s(self) -> float:
|
||||
return float(getattr(self._inner, "last_rate_limit_wait_s", 0.0) or 0.0)
|
||||
|
||||
def _log_call(self, model: str, started: float, error: str | None = None) -> None:
|
||||
"""One event per model call: how long it took and what it used, so a
|
||||
slow turn shows whether reading or writing took the time."""
|
||||
if self._oplog is None:
|
||||
return
|
||||
fields: dict[str, Any] = {
|
||||
"model": model,
|
||||
"seconds": round(time.monotonic() - started, 1),
|
||||
}
|
||||
if error is not None:
|
||||
self._oplog.emit("llm.call", cause="failed", error=error, **fields)
|
||||
return
|
||||
usage = self.last_usage
|
||||
answered = self.last_model
|
||||
if answered and answered != model:
|
||||
fields["answered_by"] = answered
|
||||
self._oplog.emit(
|
||||
"llm.call",
|
||||
cause="answered",
|
||||
input_tokens=int(usage.get("input", 0)),
|
||||
output_tokens=int(usage.get("output", 0)),
|
||||
cached_tokens=int(usage.get("cached", 0)),
|
||||
**fields,
|
||||
)
|
||||
|
||||
def budget_for(self, model: str) -> Budget:
|
||||
window, model_max_output = self._limits_for(model)
|
||||
ceiling = self._max_output_tokens
|
||||
@@ -360,17 +386,23 @@ class BudgetedClient:
|
||||
)
|
||||
if require_tool_call is None:
|
||||
require_tool_call = bool(tools) and self._enforces_tool_choice(model)
|
||||
produced = self._inner.complete(
|
||||
fitted,
|
||||
tools,
|
||||
model,
|
||||
system=system,
|
||||
max_tokens=budget.max_output,
|
||||
# Every caller that offers tools needs one called: the executor
|
||||
# finishes with finish, the planner with submit_plan or
|
||||
# ask_human, a delegation with hand_back.
|
||||
require_tool_call=require_tool_call,
|
||||
)
|
||||
started = time.monotonic()
|
||||
try:
|
||||
produced = self._inner.complete(
|
||||
fitted,
|
||||
tools,
|
||||
model,
|
||||
system=system,
|
||||
max_tokens=budget.max_output,
|
||||
# Every caller that offers tools needs one called: the executor
|
||||
# finishes with finish, the planner with submit_plan or
|
||||
# ask_human, a delegation with hand_back.
|
||||
require_tool_call=require_tool_call,
|
||||
)
|
||||
except Exception as e:
|
||||
self._log_call(model, started, error=type(e).__name__)
|
||||
raise
|
||||
self._log_call(model, started)
|
||||
reported = int(self.last_usage.get("input", 0))
|
||||
if reported > 0:
|
||||
estimated = estimate_tokens(fitted, system, tools)
|
||||
|
||||
@@ -244,6 +244,10 @@ class LiteLLMClient:
|
||||
self.last_usage = {
|
||||
"input": int(usage.get("prompt_tokens") or 0),
|
||||
"output": int(usage.get("completion_tokens") or 0),
|
||||
# Prompt tokens the server reused from its cache, when it says.
|
||||
"cached": int(
|
||||
(usage.get("prompt_tokens_details") or {}).get("cached_tokens") or 0
|
||||
),
|
||||
}
|
||||
out: list[CanonicalEvent] = []
|
||||
if message.get("content"):
|
||||
@@ -371,6 +375,7 @@ class AnthropicPassthroughClient:
|
||||
self.last_usage = {
|
||||
"input": int(usage.get("input_tokens") or 0),
|
||||
"output": int(usage.get("output_tokens") or 0),
|
||||
"cached": int(usage.get("cache_read_input_tokens") or 0),
|
||||
}
|
||||
out: list[CanonicalEvent] = []
|
||||
for block in content:
|
||||
|
||||
Reference in new issue
Block a user