Wait out LLM outages with backoff (L5: outages delay, never fail)

A 502, 503 or 504 from the LiteLLM proxy, or a dropped connection, is
retried after 5, 15 and 45 seconds before the call fails. Twice on
2026-10-07 a short burst of 502s stopped work outright: the integration
tier at 16:39 and an e2e conversation at 20:43, where the item waited on
the operator after one refused call.

Not retried: a 500 (it may be the request's fault), a timeout (it already
had its request_timeout_s), or a budget refusal. Rate limits keep their
single 20-second retry. The time spent waiting is reported with the rate
limit's (last_rate_limit_wait_s), so it isn't counted as the model's.

Tests: an outage waited out, one that outlasts the backoff, a dropped
connection, a 500 and a timeout not retried, a budget refusal not waited
on. The offline agent in test_agent_build skips the backoff (its proxy
doesn't exist).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE

nisse-dev/tests/test_agent_build.py

@@ -143,6 +143,9 @@ class FakeTransport:
 def build_offline(tmp_path: Path, origin: Path, monkeypatch: Any) -> Any:
     """Assemble the real graph with only the network boundaries faked."""
     config = config_for(tmp_path)
+    # The proxy here doesn't exist: a call to it fails at once rather than
+    # waiting out an outage's backoff (L5).
+    monkeypatch.setattr("nisse_dev.llm.OUTAGE_BACKOFF_S", ())
 
     monkeypatch.setattr(
         "nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"

nisse-dev/tests/test_llm.py

@@ -14,6 +14,7 @@ from nisse_dev.canonical import (
 from nisse_dev.envfile import load_env_file
 from nisse_dev.llm import (
     DEFAULT_MAX_OUTPUT_TOKENS,
+    OUTAGE_BACKOFF_S,
     RATE_LIMIT_RETRY_S,
     AnthropicPassthroughClient,
     BudgetExhausted,
@@ -432,3 +433,81 @@ def test_the_answering_model_is_recorded() -> None:
     )
     client.complete(EVENTS, tools=None, model="local-fast")
     assert client.last_model == "gpt-test-1"
+
+
+# --- outages delay, never fail (L5) --------------------------------------------
+
+
+def test_a_gateway_error_is_waited_out_with_backoff() -> None:
+    responses = [
+        httpx.Response(502, text="bad gateway"),
+        httpx.Response(503, text="unavailable"),
+        completion({"content": "back"}),
+    ]
+    waited: list[float] = []
+    client = make_client(lambda request: responses.pop(0), sleep=waited.append)
+    [event] = client.complete(EVENTS, tools=None, model="local/qwen")
+    assert isinstance(event, AssistantText) and event.text == "back"
+    assert waited == list(OUTAGE_BACKOFF_S[:2])
+    # Waiting isn't the model's time.
+    assert client.last_rate_limit_wait_s == sum(OUTAGE_BACKOFF_S[:2])
+
+
+def test_an_outage_that_outlasts_the_backoff_is_surfaced() -> None:
+    attempts: list[int] = []
+    waited: list[float] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        attempts.append(1)
+        return httpx.Response(502, text="bad gateway")
+
+    client = make_client(handler, sleep=waited.append)
+    with pytest.raises(UpstreamUnavailable, match="502"):
+        client.complete(EVENTS, tools=None, model="local/qwen")
+    assert len(attempts) == len(OUTAGE_BACKOFF_S) + 1
+    assert waited == list(OUTAGE_BACKOFF_S)
+
+
+def test_a_dropped_connection_is_retried() -> None:
+    calls: list[int] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(1)
+        if len(calls) == 1:
+            raise httpx.ConnectError("refused")
+        return completion({"content": "ok"})
+
+    [event] = make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
+    assert isinstance(event, AssistantText) and len(calls) == 2
+
+
+def test_a_500_or_a_timeout_is_not_retried() -> None:
+    # A 500 may be the request's fault; a timeout already had its full
+    # request_timeout_s.
+    for failure in (
+        lambda request: httpx.Response(500, text="boom"),
+        lambda request: (_ for _ in ()).throw(httpx.ReadTimeout("slow")),
+    ):
+        calls: list[int] = []
+
+        def handler(
+            request: httpx.Request, failure=failure, calls=calls
+        ) -> httpx.Response:
+            calls.append(1)
+            return failure(request)
+
+        with pytest.raises(LLMError):
+            make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
+        assert len(calls) == 1
+
+
+def test_a_budget_refusal_is_not_waited_on() -> None:
+    calls: list[int] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(1)
+        return httpx.Response(503, text="Budget has been exceeded! Current cost: 5.1")
+
+    with pytest.raises(BudgetExhausted):
+        make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
+    assert len(calls) == 1