Enforce the token cap, and treat a spent budget as an interruption
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 51s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 51s
Exceeding a LiteLLM spend limit takes external models away for the rest of the day, which makes three existing gaps consequential rather than untidy. - A13 calls the delegation caps hard on "turns, tokens, and wall clock", and the module docstring repeated it, but the token cap was decoration: nothing counted, and result.tokens was always zero. It is now metered from the usage the provider reports, so a confused delegate cannot spend a day's allowance. A5 keeps the *money* cap on the LiteLLM key, which is a different thing; this is a scope cap - The normalized route sent no ceiling at all, so truncation detection there had nothing to detect against. Both routes now send one - A budget refusal is recognised distinctly. It is not a rate limit: retrying cannot help, so the planner no longer burns a second call to fail identically And a delegation cut off by an exhausted budget is now "interrupted" rather than "failed", which preserves its branch and leaves its intent pending. The work on that branch may be perfectly good; discarding it because the budget ran out would throw away work for a reason that has nothing to do with the work. A3's resume rule already knows how to pick such a delegation back up. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
249bcb10fc
commit
8cfb45aed3
5 files changed
+232
-6
No files matched your search
@@ -54,6 +54,26 @@ def max_output_tokens(env: Mapping[str, str] = os.environ) -> int:
|
||||
return int(raw) if raw else DEFAULT_MAX_OUTPUT_TOKENS
|
||||
|
||||
|
||||
class BudgetExhausted(LLMError):
|
||||
"""The proxy refused because a spend limit is gone.
|
||||
|
||||
Distinct from a rate limit, which clears in a minute: an exhausted daily
|
||||
budget takes external models away until it resets, so retrying wastes
|
||||
time and the operator needs telling rather than the loop quietly failing.
|
||||
A5 keeps the budget itself at the proxy; recognising its refusal is ours.
|
||||
"""
|
||||
|
||||
|
||||
def _looks_like_budget_refusal(text: str) -> bool:
|
||||
# Pattern-matched rather than contracted: LiteLLM's wording is not a
|
||||
# documented interface, so this errs toward missing one rather than
|
||||
# misreporting an unrelated failure as a budget problem.
|
||||
lowered = text.lower()
|
||||
return "budget" in lowered and (
|
||||
"exceed" in lowered or "exhaust" in lowered or "limit" in lowered
|
||||
)
|
||||
|
||||
|
||||
class TruncatedResponse(LLMError):
|
||||
"""The model was cut off mid-answer.
|
||||
|
||||
@@ -114,6 +134,7 @@ class LiteLLMClient:
|
||||
timeout_s: float = 120.0,
|
||||
transport: httpx.BaseTransport | None = None,
|
||||
sleep: Any = time.sleep,
|
||||
max_tokens: int | None = None,
|
||||
) -> None:
|
||||
self._http = httpx.Client(
|
||||
base_url=base_url.rstrip("/"),
|
||||
@@ -123,6 +144,11 @@ class LiteLLMClient:
|
||||
)
|
||||
self._renderer = OpenAIChatRenderer()
|
||||
self._sleep = sleep
|
||||
self._max_tokens = max_tokens or max_output_tokens()
|
||||
# Set after each call so a caller can meter itself (A13 caps);
|
||||
# a side channel rather than a return value, so the LLMClient
|
||||
# protocol stays the same for every implementation.
|
||||
self.last_usage: dict[str, int] = {}
|
||||
|
||||
@classmethod
|
||||
def from_env(cls, env: Mapping[str, str] = os.environ) -> "LiteLLMClient":
|
||||
@@ -140,10 +166,18 @@ class LiteLLMClient:
|
||||
system: str | None = None,
|
||||
) -> list[CanonicalEvent]:
|
||||
body = self._renderer.render(events, tools, model, system=system)
|
||||
# A ceiling on this route too: without one a runaway turn is bounded
|
||||
# only by the provider's default, and the same truncation check has
|
||||
# nothing to check against.
|
||||
body["max_tokens"] = self._max_tokens
|
||||
response = _post_with_rate_limit_retry(
|
||||
self._http, "/chat/completions", body, self._sleep
|
||||
)
|
||||
if response.status_code != 200:
|
||||
if _looks_like_budget_refusal(response.text):
|
||||
raise BudgetExhausted(
|
||||
f"the proxy refused on budget grounds: {response.text[:300]}"
|
||||
)
|
||||
raise LLMError(
|
||||
f"LLM request returned {response.status_code}: {response.text[:500]}"
|
||||
)
|
||||
@@ -163,6 +197,11 @@ class LiteLLMClient:
|
||||
"raise NISSE_MAX_OUTPUT_TOKENS"
|
||||
)
|
||||
produced_by = data.get("model") or requested_model
|
||||
usage = data.get("usage") or {}
|
||||
self.last_usage = {
|
||||
"input": int(usage.get("prompt_tokens") or 0),
|
||||
"output": int(usage.get("completion_tokens") or 0),
|
||||
}
|
||||
out: list[CanonicalEvent] = []
|
||||
if message.get("content"):
|
||||
out.append(AssistantText(text=message["content"], model=produced_by))
|
||||
@@ -229,6 +268,7 @@ class AnthropicPassthroughClient:
|
||||
self._renderer = AnthropicMessagesRenderer()
|
||||
self._max_tokens = max_tokens or max_output_tokens()
|
||||
self._sleep = sleep
|
||||
self.last_usage: dict[str, int] = {}
|
||||
|
||||
@classmethod
|
||||
def from_env(
|
||||
@@ -254,6 +294,10 @@ class AnthropicPassthroughClient:
|
||||
self._http, "/v1/messages", body, self._sleep
|
||||
)
|
||||
if response.status_code != 200:
|
||||
if _looks_like_budget_refusal(response.text):
|
||||
raise BudgetExhausted(
|
||||
f"the proxy refused on budget grounds: {response.text[:300]}"
|
||||
)
|
||||
raise LLMError(
|
||||
f"passthrough returned {response.status_code}: {response.text[:500]}"
|
||||
)
|
||||
@@ -271,6 +315,11 @@ class AnthropicPassthroughClient:
|
||||
if not isinstance(content, list):
|
||||
raise LLMError(f"malformed passthrough response: {str(data)[:300]}")
|
||||
produced_by = data.get("model") or requested_model
|
||||
usage = data.get("usage") or {}
|
||||
self.last_usage = {
|
||||
"input": int(usage.get("input_tokens") or 0),
|
||||
"output": int(usage.get("output_tokens") or 0),
|
||||
}
|
||||
out: list[CanonicalEvent] = []
|
||||
for block in content:
|
||||
kind = block.get("type")
|
||||
|
||||
Reference in new issue
Block a user