diff --git a/nisse-dev/src/nisse_dev/delegation.py b/nisse-dev/src/nisse_dev/delegation.py index aba2bbe..9940904 100644 --- a/nisse-dev/src/nisse_dev/delegation.py +++ b/nisse-dev/src/nisse_dev/delegation.py @@ -36,7 +36,7 @@ from nisse_dev.canonical import ( ToolResult, ) from nisse_dev.git import GitError, GitRepo -from nisse_dev.llm import LLMClient, LLMError +from nisse_dev.llm import BudgetExhausted, LLMClient, LLMError from nisse_dev.reconcile import Reconciler from nisse_dev.state import Intent, StateStore from nisse_dev.tools import EXECUTOR_TOOLS, MUTATING_TOOLS, Workspace @@ -129,7 +129,7 @@ class Handback: @dataclass class DelegationResult: - status: Literal["handback", "capped", "failed"] + status: Literal["handback", "capped", "failed", "interrupted"] note: str handback: Handback | None = None branch: str | None = None @@ -137,6 +137,17 @@ class DelegationResult: tokens: int = 0 duration_s: float = 0.0 + @property + def interrupted(self) -> bool: + """Stopped by something external rather than by the delegate. + + The work on the branch may be perfectly good, so it is preserved and + the intent left pending: the next boot treats it exactly like a crash + mid-delegation (A3) and resumes if the cut was clean. Discarding here + would throw away work because the budget ran out. + """ + return self.status == "interrupted" + @property def usable(self) -> bool: """Only a successful handback may be built upon. A cap breach or a @@ -212,6 +223,11 @@ class Delegation: exit_status=result.status, ) ) + if result.interrupted: + # Leave the branch and the intent alone: this delegation is not + # finished, it was cut off, and A3's resume rule is exactly the + # machinery for picking it back up. + return result if not result.usable: # Abort discards the branch: the caller must never resume on top # of a partially-edited tree (A13). @@ -294,6 +310,7 @@ class Delegation: ) touched: set[str] = set() did_work = False + spent = 0 for turn in range(self._caps.turns): elapsed = self._now() - started_at @@ -304,6 +321,7 @@ class Delegation: f"after {turn} turns", branch=branch, artifacts=sorted(touched), + tokens=spent, ) try: produced = self._client.complete( @@ -312,9 +330,35 @@ class Delegation: model=self._model, system=system, ) + except BudgetExhausted as exc: + return DelegationResult( + "interrupted", + f"external models are unavailable: {exc}", + branch=branch, + artifacts=sorted(touched), + tokens=spent, + ) except LLMError as exc: return DelegationResult( - "failed", f"delegate call failed: {exc}", branch=branch + "failed", + f"delegate call failed: {exc}", + branch=branch, + tokens=spent, + ) + # Metered from what the provider reported, not estimated. The cap + # is a scope limit (A13), distinct from the money cap A5 keeps on + # the LiteLLM key — but an unbounded delegation can still exhaust + # a daily budget and take external models away until it resets. + usage = getattr(self._client, "last_usage", {}) or {} + spent += int(usage.get("input", 0)) + int(usage.get("output", 0)) + if spent >= self._caps.tokens: + return DelegationResult( + "capped", + f"token cap of {self._caps.tokens} reached ({spent} spent) " + f"after {turn + 1} turns", + branch=branch, + artifacts=sorted(touched), + tokens=spent, ) calls: list[ToolCall] = [] @@ -336,6 +380,7 @@ class Delegation: "delegate produced neither a tool call nor text", branch=branch, artifacts=sorted(touched), + tokens=spent, ) for call in calls: @@ -385,6 +430,7 @@ class Delegation: handback=handback if handback.succeeded else None, branch=branch, artifacts=sorted(touched), + tokens=spent, ) outcome = self._workspace.dispatch(call.tool, call.arguments) @@ -411,6 +457,7 @@ class Delegation: f"turn cap of {self._caps.turns} reached without a handback", branch=branch, artifacts=sorted(touched), + tokens=spent, ) def _commit_turn(self, turn: int) -> None: diff --git a/nisse-dev/src/nisse_dev/llm.py b/nisse-dev/src/nisse_dev/llm.py index adc34f9..7f3a39a 100644 --- a/nisse-dev/src/nisse_dev/llm.py +++ b/nisse-dev/src/nisse_dev/llm.py @@ -54,6 +54,26 @@ def max_output_tokens(env: Mapping[str, str] = os.environ) -> int: return int(raw) if raw else DEFAULT_MAX_OUTPUT_TOKENS +class BudgetExhausted(LLMError): + """The proxy refused because a spend limit is gone. + + Distinct from a rate limit, which clears in a minute: an exhausted daily + budget takes external models away until it resets, so retrying wastes + time and the operator needs telling rather than the loop quietly failing. + A5 keeps the budget itself at the proxy; recognising its refusal is ours. + """ + + +def _looks_like_budget_refusal(text: str) -> bool: + # Pattern-matched rather than contracted: LiteLLM's wording is not a + # documented interface, so this errs toward missing one rather than + # misreporting an unrelated failure as a budget problem. + lowered = text.lower() + return "budget" in lowered and ( + "exceed" in lowered or "exhaust" in lowered or "limit" in lowered + ) + + class TruncatedResponse(LLMError): """The model was cut off mid-answer. @@ -114,6 +134,7 @@ class LiteLLMClient: timeout_s: float = 120.0, transport: httpx.BaseTransport | None = None, sleep: Any = time.sleep, + max_tokens: int | None = None, ) -> None: self._http = httpx.Client( base_url=base_url.rstrip("/"), @@ -123,6 +144,11 @@ class LiteLLMClient: ) self._renderer = OpenAIChatRenderer() self._sleep = sleep + self._max_tokens = max_tokens or max_output_tokens() + # Set after each call so a caller can meter itself (A13 caps); + # a side channel rather than a return value, so the LLMClient + # protocol stays the same for every implementation. + self.last_usage: dict[str, int] = {} @classmethod def from_env(cls, env: Mapping[str, str] = os.environ) -> "LiteLLMClient": @@ -140,10 +166,18 @@ class LiteLLMClient: system: str | None = None, ) -> list[CanonicalEvent]: body = self._renderer.render(events, tools, model, system=system) + # A ceiling on this route too: without one a runaway turn is bounded + # only by the provider's default, and the same truncation check has + # nothing to check against. + body["max_tokens"] = self._max_tokens response = _post_with_rate_limit_retry( self._http, "/chat/completions", body, self._sleep ) if response.status_code != 200: + if _looks_like_budget_refusal(response.text): + raise BudgetExhausted( + f"the proxy refused on budget grounds: {response.text[:300]}" + ) raise LLMError( f"LLM request returned {response.status_code}: {response.text[:500]}" ) @@ -163,6 +197,11 @@ class LiteLLMClient: "raise NISSE_MAX_OUTPUT_TOKENS" ) produced_by = data.get("model") or requested_model + usage = data.get("usage") or {} + self.last_usage = { + "input": int(usage.get("prompt_tokens") or 0), + "output": int(usage.get("completion_tokens") or 0), + } out: list[CanonicalEvent] = [] if message.get("content"): out.append(AssistantText(text=message["content"], model=produced_by)) @@ -229,6 +268,7 @@ class AnthropicPassthroughClient: self._renderer = AnthropicMessagesRenderer() self._max_tokens = max_tokens or max_output_tokens() self._sleep = sleep + self.last_usage: dict[str, int] = {} @classmethod def from_env( @@ -254,6 +294,10 @@ class AnthropicPassthroughClient: self._http, "/v1/messages", body, self._sleep ) if response.status_code != 200: + if _looks_like_budget_refusal(response.text): + raise BudgetExhausted( + f"the proxy refused on budget grounds: {response.text[:300]}" + ) raise LLMError( f"passthrough returned {response.status_code}: {response.text[:500]}" ) @@ -271,6 +315,11 @@ class AnthropicPassthroughClient: if not isinstance(content, list): raise LLMError(f"malformed passthrough response: {str(data)[:300]}") produced_by = data.get("model") or requested_model + usage = data.get("usage") or {} + self.last_usage = { + "input": int(usage.get("input_tokens") or 0), + "output": int(usage.get("output_tokens") or 0), + } out: list[CanonicalEvent] = [] for block in content: kind = block.get("type") diff --git a/nisse-dev/src/nisse_dev/planner.py b/nisse-dev/src/nisse_dev/planner.py index be599b0..f0b4e86 100644 --- a/nisse-dev/src/nisse_dev/planner.py +++ b/nisse-dev/src/nisse_dev/planner.py @@ -13,7 +13,7 @@ from typing import Any from nisse_dev.canonical import AssistantText, ToolCall, ToolResult from nisse_dev.control import AskHuman, PlanningError, SubmitPlan -from nisse_dev.llm import LLMClient, LLMError, RoleMap +from nisse_dev.llm import BudgetExhausted, LLMClient, LLMError, RoleMap from nisse_dev.state import StateStore # Plans may be executed by a materially weaker model, which cannot @@ -125,6 +125,11 @@ class LLMPlanningAgent: output is observable, never the model's own claim).""" try: return self._turn(model, system), None + except BudgetExhausted: + # Retrying cannot help: the budget will not reappear within a + # turn. Surface it now so the operator is told rather than the + # loop spending a second call to fail identically. + raise except LLMError as exc: return None, str(exc) diff --git a/nisse-dev/tests/test_delegation.py b/nisse-dev/tests/test_delegation.py index 53dcdc8..e42b4ce 100644 --- a/nisse-dev/tests/test_delegation.py +++ b/nisse-dev/tests/test_delegation.py @@ -21,7 +21,7 @@ from nisse_dev.delegation import ( delegation_reconciler, ) from nisse_dev.git import GitRepo -from nisse_dev.llm import LLMError +from nisse_dev.llm import BudgetExhausted, LLMError from nisse_dev.oplog import OpLog from nisse_dev.state import StateStore from nisse_dev.tools import Workspace @@ -507,3 +507,110 @@ def test_recover_leaves_an_unclean_cut_for_the_boot_gate( assert build(ScriptedClient([]), store, ws, repo).recover(lambda _m: None) is None # Still pending, so reconcile_on_boot gates rather than blind-resuming. assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT] + + +# --- the token cap, which A13 calls hard and was decoration ------------- + + +class MeteredClient(ScriptedClient): + """A client that reports usage, as the real ones do.""" + + def __init__(self, turns: list, per_call: int = 1000) -> None: + super().__init__(turns) + self._per_call = per_call + self.last_usage: dict[str, int] = {} + + def complete(self, *args: Any, **kwargs: Any): + produced = super().complete(*args, **kwargs) + self.last_usage = {"input": self._per_call, "output": self._per_call} + return produced + + +def test_token_cap_stops_a_delegation( + store: StateStore, ws: Workspace, repo: GitRepo +) -> None: + # An unbounded delegation can exhaust a daily budget and take external + # models away until it resets, so the cap has to be real. + client = MeteredClient( + [[call("list_dir", path=".")] for _ in range(20)], per_call=600 + ) + result = build(client, store, ws, repo, caps=Caps(turns=20, tokens=2000)).run( + brief="b", branch="d" + ) + + assert result.status == "capped" + assert "token cap of 2000" in result.note + # 600 in + 600 out per call: stops on the second turn, not the twentieth. + assert len(client.calls) == 2 + + +def test_tokens_spent_are_reported_not_left_at_zero( + store: StateStore, ws: Workspace, repo: GitRepo +) -> None: + client = MeteredClient( + [[call("write_file", path="a.py", content="x")], [hand_back()]], per_call=500 + ) + result = build(client, store, ws, repo).run(brief="b", branch="d") + + assert result.usable + assert result.tokens == 2000 # two calls, 1000 each + + completed = next( + e.event for e in store.read_events() if isinstance(e.event, DelegationCompleted) + ) + # A16 evaluates cost against outcome, which needs a real number. + assert completed.tokens == 2000 + + +def test_a_client_without_usage_still_works( + store: StateStore, ws: Workspace, repo: GitRepo +) -> None: + # Scripted and third-party clients need not report usage; the cap simply + # does not bind rather than crashing the delegation. + client = ScriptedClient( + [[call("write_file", path="a.py", content="x")], [hand_back()]] + ) + assert build(client, store, ws, repo).run(brief="b", branch="d").usable + + +def test_budget_exhaustion_preserves_the_work_for_later( + store: StateStore, ws: Workspace, repo: GitRepo +) -> None: + """An outage is not a bad delegate. + + The branch may hold perfectly good work, so it is kept and the intent + left pending — the next boot treats it exactly like a crash mid-delegation + and resumes if the cut was clean. Discarding would throw away work + because the budget ran out. + """ + + class BrokeClient(ScriptedClient): + def complete(self, *args: Any, **kwargs: Any): + if self.calls: + raise BudgetExhausted("daily budget exhausted") + return super().complete(*args, **kwargs) + + client = BrokeClient([[call("write_file", path="progress.py", content="real")]]) + result = build(client, store, ws, repo).run(brief="b", branch="d") + + assert result.status == "interrupted" + assert result.interrupted and not result.usable + # The work survives, on its branch. + assert repo.current_branch() == "d" + assert (repo.root / "progress.py").exists() + # And the intent stays pending, which is what A3's resume rule keys on. + assert [i.kind for i in store.pending_intents()] == [DELEGATION_INTENT] + + +def test_an_ordinary_failure_still_discards( + store: StateStore, ws: Workspace, repo: GitRepo +) -> None: + # Only external interruptions are preserved; a delegate that made a mess + # is still cleaned up (A13). + client = ScriptedClient([[hand_back(succeeded=False, changed="cannot")]]) + result = build(client, store, ws, repo).run(brief="b", branch="d") + + assert result.status == "failed" + assert not result.interrupted + assert repo.current_branch() == "main" + assert store.pending_intents() == [] diff --git a/nisse-dev/tests/test_planner.py b/nisse-dev/tests/test_planner.py index 9dcce14..0334242 100644 --- a/nisse-dev/tests/test_planner.py +++ b/nisse-dev/tests/test_planner.py @@ -20,7 +20,7 @@ from nisse_dev.control import ( PlanningError, SubmitPlan, ) -from nisse_dev.llm import RoleMap +from nisse_dev.llm import BudgetExhausted, RoleMap from nisse_dev.oplog import OpLog from nisse_dev.planner import PLANNER_TOOLS, LLMPlanningAgent from nisse_dev.state import StateStore @@ -279,3 +279,21 @@ def test_rejection_feedback_reaches_the_planner_via_the_log( isinstance(e, UserMessage) and "too vague" in e.text for e in replan_context ) assert "v2" in transport.outbound[-1].text + + +def test_budget_exhaustion_is_not_retried(store: StateStore) -> None: + """A budget will not reappear within a turn, so a retry buys nothing and + delays telling the operator.""" + + class BrokeClient: + def __init__(self) -> None: + self.calls = 0 + + def complete(self, *args: Any, **kwargs: Any): + self.calls += 1 + raise BudgetExhausted("daily budget exhausted") + + client = BrokeClient() + with pytest.raises(BudgetExhausted): + LLMPlanningAgent(client, store, ROLES).plan("arch", "demo") + assert client.calls == 1 # not two