Enforce the token cap, and treat a spent budget as an interruption
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 51s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 51s
Exceeding a LiteLLM spend limit takes external models away for the rest of the day, which makes three existing gaps consequential rather than untidy. - A13 calls the delegation caps hard on "turns, tokens, and wall clock", and the module docstring repeated it, but the token cap was decoration: nothing counted, and result.tokens was always zero. It is now metered from the usage the provider reports, so a confused delegate cannot spend a day's allowance. A5 keeps the *money* cap on the LiteLLM key, which is a different thing; this is a scope cap - The normalized route sent no ceiling at all, so truncation detection there had nothing to detect against. Both routes now send one - A budget refusal is recognised distinctly. It is not a rate limit: retrying cannot help, so the planner no longer burns a second call to fail identically And a delegation cut off by an exhausted budget is now "interrupted" rather than "failed", which preserves its branch and leaves its intent pending. The work on that branch may be perfectly good; discarding it because the budget ran out would throw away work for a reason that has nothing to do with the work. A3's resume rule already knows how to pick such a delegation back up. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
249bcb10fc
commit
8cfb45aed3
5 files changed
+232
-6
No files matched your search
@@ -21,7 +21,7 @@ from nisse_dev.delegation import (
|
||||
delegation_reconciler,
|
||||
)
|
||||
from nisse_dev.git import GitRepo
|
||||
from nisse_dev.llm import LLMError
|
||||
from nisse_dev.llm import BudgetExhausted, LLMError
|
||||
from nisse_dev.oplog import OpLog
|
||||
from nisse_dev.state import StateStore
|
||||
from nisse_dev.tools import Workspace
|
||||
@@ -507,3 +507,110 @@ def test_recover_leaves_an_unclean_cut_for_the_boot_gate(
|
||||
assert build(ScriptedClient([]), store, ws, repo).recover(lambda _m: None) is None
|
||||
# Still pending, so reconcile_on_boot gates rather than blind-resuming.
|
||||
assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT]
|
||||
|
||||
|
||||
# --- the token cap, which A13 calls hard and was decoration -------------
|
||||
|
||||
|
||||
class MeteredClient(ScriptedClient):
|
||||
"""A client that reports usage, as the real ones do."""
|
||||
|
||||
def __init__(self, turns: list, per_call: int = 1000) -> None:
|
||||
super().__init__(turns)
|
||||
self._per_call = per_call
|
||||
self.last_usage: dict[str, int] = {}
|
||||
|
||||
def complete(self, *args: Any, **kwargs: Any):
|
||||
produced = super().complete(*args, **kwargs)
|
||||
self.last_usage = {"input": self._per_call, "output": self._per_call}
|
||||
return produced
|
||||
|
||||
|
||||
def test_token_cap_stops_a_delegation(
|
||||
store: StateStore, ws: Workspace, repo: GitRepo
|
||||
) -> None:
|
||||
# An unbounded delegation can exhaust a daily budget and take external
|
||||
# models away until it resets, so the cap has to be real.
|
||||
client = MeteredClient(
|
||||
[[call("list_dir", path=".")] for _ in range(20)], per_call=600
|
||||
)
|
||||
result = build(client, store, ws, repo, caps=Caps(turns=20, tokens=2000)).run(
|
||||
brief="b", branch="d"
|
||||
)
|
||||
|
||||
assert result.status == "capped"
|
||||
assert "token cap of 2000" in result.note
|
||||
# 600 in + 600 out per call: stops on the second turn, not the twentieth.
|
||||
assert len(client.calls) == 2
|
||||
|
||||
|
||||
def test_tokens_spent_are_reported_not_left_at_zero(
|
||||
store: StateStore, ws: Workspace, repo: GitRepo
|
||||
) -> None:
|
||||
client = MeteredClient(
|
||||
[[call("write_file", path="a.py", content="x")], [hand_back()]], per_call=500
|
||||
)
|
||||
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
||||
|
||||
assert result.usable
|
||||
assert result.tokens == 2000 # two calls, 1000 each
|
||||
|
||||
completed = next(
|
||||
e.event for e in store.read_events() if isinstance(e.event, DelegationCompleted)
|
||||
)
|
||||
# A16 evaluates cost against outcome, which needs a real number.
|
||||
assert completed.tokens == 2000
|
||||
|
||||
|
||||
def test_a_client_without_usage_still_works(
|
||||
store: StateStore, ws: Workspace, repo: GitRepo
|
||||
) -> None:
|
||||
# Scripted and third-party clients need not report usage; the cap simply
|
||||
# does not bind rather than crashing the delegation.
|
||||
client = ScriptedClient(
|
||||
[[call("write_file", path="a.py", content="x")], [hand_back()]]
|
||||
)
|
||||
assert build(client, store, ws, repo).run(brief="b", branch="d").usable
|
||||
|
||||
|
||||
def test_budget_exhaustion_preserves_the_work_for_later(
|
||||
store: StateStore, ws: Workspace, repo: GitRepo
|
||||
) -> None:
|
||||
"""An outage is not a bad delegate.
|
||||
|
||||
The branch may hold perfectly good work, so it is kept and the intent
|
||||
left pending — the next boot treats it exactly like a crash mid-delegation
|
||||
and resumes if the cut was clean. Discarding would throw away work
|
||||
because the budget ran out.
|
||||
"""
|
||||
|
||||
class BrokeClient(ScriptedClient):
|
||||
def complete(self, *args: Any, **kwargs: Any):
|
||||
if self.calls:
|
||||
raise BudgetExhausted("daily budget exhausted")
|
||||
return super().complete(*args, **kwargs)
|
||||
|
||||
client = BrokeClient([[call("write_file", path="progress.py", content="real")]])
|
||||
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
||||
|
||||
assert result.status == "interrupted"
|
||||
assert result.interrupted and not result.usable
|
||||
# The work survives, on its branch.
|
||||
assert repo.current_branch() == "d"
|
||||
assert (repo.root / "progress.py").exists()
|
||||
# And the intent stays pending, which is what A3's resume rule keys on.
|
||||
assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT]
|
||||
|
||||
|
||||
def test_an_ordinary_failure_still_discards(
|
||||
store: StateStore, ws: Workspace, repo: GitRepo
|
||||
) -> None:
|
||||
# Only external interruptions are preserved; a delegate that made a mess
|
||||
# is still cleaned up (A13).
|
||||
client = ScriptedClient([[hand_back(succeeded=False, changed="cannot")]])
|
||||
result = build(client, store, ws, repo).run(brief="b", branch="d")
|
||||
|
||||
assert result.status == "failed"
|
||||
assert not result.interrupted
|
||||
assert repo.current_branch() == "main"
|
||||
assert store.pending_intents() == []
|
||||
Reference in new issue
Block a user