From ad8a07fcb7deae1db2a667f4386ad24083eece47 Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Wed, 7 Oct 2026 21:11:45 +0000 Subject: [PATCH] Wait out LLM outages with backoff (L5: outages delay, never fail) A 502, 503 or 504 from the LiteLLM proxy, or a dropped connection, is retried after 5, 15 and 45 seconds before the call fails. Twice on 2026-10-07 a short burst of 502s stopped work outright: the integration tier at 16:39 and an e2e conversation at 20:43, where the item waited on the operator after one refused call. Not retried: a 500 (it may be the request's fault), a timeout (it already had its request_timeout_s), or a budget refusal. Rate limits keep their single 20-second retry. The time spent waiting is reported with the rate limit's (last_rate_limit_wait_s), so it isn't counted as the model's. Tests: an outage waited out, one that outlasts the backoff, a dropped connection, a 500 and a timeout not retried, a budget refusal not waited on. The offline agent in test_agent_build skips the backoff (its proxy doesn't exist). Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE --- nisse-dev/src/nisse_dev/llm.py | 42 ++++++++++++--- nisse-dev/tests/test_agent_build.py | 3 ++ nisse-dev/tests/test_llm.py | 79 +++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+), 8 deletions(-) diff --git a/nisse-dev/src/nisse_dev/llm.py b/nisse-dev/src/nisse_dev/llm.py index 562de2f..432f4f7 100644 --- a/nisse-dev/src/nisse_dev/llm.py +++ b/nisse-dev/src/nisse_dev/llm.py @@ -39,6 +39,13 @@ class LLMError(RuntimeError): RATE_LIMIT_STATUS = 429 RATE_LIMIT_RETRY_S = 20.0 +# The proxy, or what's behind it, briefly gone: a 502/503/504 or a dropped +# connection. Outages delay, never fail (L5): waited out with backoff before +# the work is stopped. Not a 500 (it may be the request) and not a timeout +# (a hung model has had its request_timeout_s already). +OUTAGE_STATUSES = frozenset({502, 503, 504}) +OUTAGE_BACKOFF_S = (5.0, 15.0, 45.0) + # Generous by default. Output is billed by what is produced, not by the # ceiling, and a ceiling too low does not fail loudly — it truncates a tool # call mid-argument and hands back something that parses. Measured: Opus @@ -381,20 +388,39 @@ class AnthropicPassthroughClient: def _post_with_rate_limit_retry( http: httpx.Client, path: str, body: dict[str, Any], sleep: Any ) -> tuple[httpx.Response, float]: - """The response, and how long was spent waiting out a rate limit.""" + """The response, and how long was spent waiting: out a rate limit (once) + or an outage (with backoff), not on the model.""" waited = 0.0 - for attempt in (1, 2): + rate_limited = False + outages = iter(OUTAGE_BACKOFF_S) + while True: try: response = http.post(path, json=body) + except httpx.TimeoutException as exc: + raise UpstreamUnavailable(f"LLM request failed: {exc}") from exc except httpx.HTTPError as exc: # Could not reach the proxy at all: nothing about the request was # wrong, so callers holding work in progress should keep it. - raise UpstreamUnavailable(f"LLM request failed: {exc}") from exc - if response.status_code != RATE_LIMIT_STATUS or attempt == 2: - return response, waited - sleep(RATE_LIMIT_RETRY_S) - waited += RATE_LIMIT_RETRY_S - raise AssertionError("unreachable") + pause = next(outages, None) + if pause is None: + raise UpstreamUnavailable(f"LLM request failed: {exc}") from exc + sleep(pause) + waited += pause + continue + if response.status_code == RATE_LIMIT_STATUS and not rate_limited: + rate_limited = True + sleep(RATE_LIMIT_RETRY_S) + waited += RATE_LIMIT_RETRY_S + continue + if response.status_code in OUTAGE_STATUSES and not _looks_like_budget_refusal( + response.text + ): + pause = next(outages, None) + if pause is not None: + sleep(pause) + waited += pause + continue + return response, waited # Model families we have a native renderer adapter for. L9: delegations take diff --git a/nisse-dev/tests/test_agent_build.py b/nisse-dev/tests/test_agent_build.py index d11a050..831f185 100644 --- a/nisse-dev/tests/test_agent_build.py +++ b/nisse-dev/tests/test_agent_build.py @@ -143,6 +143,9 @@ class FakeTransport: def build_offline(tmp_path: Path, origin: Path, monkeypatch: Any) -> Any: """Assemble the real graph with only the network boundaries faked.""" config = config_for(tmp_path) + # The proxy here doesn't exist: a call to it fails at once rather than + # waiting out an outage's backoff (L5). + monkeypatch.setattr("nisse_dev.llm.OUTAGE_BACKOFF_S", ()) monkeypatch.setattr( "nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing" diff --git a/nisse-dev/tests/test_llm.py b/nisse-dev/tests/test_llm.py index a50656e..b4c22e3 100644 --- a/nisse-dev/tests/test_llm.py +++ b/nisse-dev/tests/test_llm.py @@ -14,6 +14,7 @@ from nisse_dev.canonical import ( from nisse_dev.envfile import load_env_file from nisse_dev.llm import ( DEFAULT_MAX_OUTPUT_TOKENS, + OUTAGE_BACKOFF_S, RATE_LIMIT_RETRY_S, AnthropicPassthroughClient, BudgetExhausted, @@ -432,3 +433,81 @@ def test_the_answering_model_is_recorded() -> None: ) client.complete(EVENTS, tools=None, model="local-fast") assert client.last_model == "gpt-test-1" + + +# --- outages delay, never fail (L5) -------------------------------------------- + + +def test_a_gateway_error_is_waited_out_with_backoff() -> None: + responses = [ + httpx.Response(502, text="bad gateway"), + httpx.Response(503, text="unavailable"), + completion({"content": "back"}), + ] + waited: list[float] = [] + client = make_client(lambda request: responses.pop(0), sleep=waited.append) + [event] = client.complete(EVENTS, tools=None, model="local/qwen") + assert isinstance(event, AssistantText) and event.text == "back" + assert waited == list(OUTAGE_BACKOFF_S[:2]) + # Waiting isn't the model's time. + assert client.last_rate_limit_wait_s == sum(OUTAGE_BACKOFF_S[:2]) + + +def test_an_outage_that_outlasts_the_backoff_is_surfaced() -> None: + attempts: list[int] = [] + waited: list[float] = [] + + def handler(request: httpx.Request) -> httpx.Response: + attempts.append(1) + return httpx.Response(502, text="bad gateway") + + client = make_client(handler, sleep=waited.append) + with pytest.raises(UpstreamUnavailable, match="502"): + client.complete(EVENTS, tools=None, model="local/qwen") + assert len(attempts) == len(OUTAGE_BACKOFF_S) + 1 + assert waited == list(OUTAGE_BACKOFF_S) + + +def test_a_dropped_connection_is_retried() -> None: + calls: list[int] = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(1) + if len(calls) == 1: + raise httpx.ConnectError("refused") + return completion({"content": "ok"}) + + [event] = make_client(handler).complete(EVENTS, tools=None, model="local/qwen") + assert isinstance(event, AssistantText) and len(calls) == 2 + + +def test_a_500_or_a_timeout_is_not_retried() -> None: + # A 500 may be the request's fault; a timeout already had its full + # request_timeout_s. + for failure in ( + lambda request: httpx.Response(500, text="boom"), + lambda request: (_ for _ in ()).throw(httpx.ReadTimeout("slow")), + ): + calls: list[int] = [] + + def handler( + request: httpx.Request, failure=failure, calls=calls + ) -> httpx.Response: + calls.append(1) + return failure(request) + + with pytest.raises(LLMError): + make_client(handler).complete(EVENTS, tools=None, model="local/qwen") + assert len(calls) == 1 + + +def test_a_budget_refusal_is_not_waited_on() -> None: + calls: list[int] = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(1) + return httpx.Response(503, text="Budget has been exceeded! Current cost: 5.1") + + with pytest.raises(BudgetExhausted): + make_client(handler).complete(EVENTS, tools=None, model="local/qwen") + assert len(calls) == 1 -- 2.54.0