Merge pull request 'Wait out LLM outages with backoff (L5)' (#116) from llm-retry into main

A 502, 503 or 504 from the LiteLLM proxy, or a dropped connection, is
retried after 5, 15 and 45 seconds before the call fails. Twice on
2026-10-07 a short burst of 502s stopped work outright: the integration
tier at 16:39 and an e2e conversation at 20:43, where the item waited on
the operator after one refused call.

Not retried: a 500 (it may be the request's fault), a timeout (it already
had its request_timeout_s), or a budget refusal. Rate limits keep their
single 20-second retry. The time spent waiting is reported with the rate
limit's (last_rate_limit_wait_s), so it isn't counted as the model's.

Tests: an outage waited out, one that outlasts the backoff, a dropped
connection, a 500 and a timeout not retried, a budget refusal not waited
on. The offline agent in test_agent_build skips the backoff (its proxy
doesn't exist).

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE

.gitea/workflows/ci.yml

@@ -55,20 +55,6 @@ jobs:
           NISSE_OPERATOR_TOKEN: ${{ secrets.NISSE_OPERATOR_TOKEN }}
         run: uv run pytest -m "integration and not e2e" -rs
 
-  image:
-    # The agent's image builds (Dockerfile), so a broken one shows before a
-    # release does. On PRs only; nothing is pushed.
-    if: github.event_name != 'push'
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-
-      - name: Build the agent and test images
-        run: |
-          docker build --target agent -t nisse-dev:ci .
-          docker build --target test -t nisse-dev:ci-test .
-          docker run --rm nisse-dev:ci 2>&1 | grep -q "cannot start: no config file"
-
   e2e:
     # The end-to-end suite (scripts/e2e.sh): the whole agent against real
     # services, every role on a local model, so it spends nothing but keeps

.gitea/workflows/release.yml

@@ -1,35 +1,63 @@
 name: Release
-# A tag v* publishes the agent's image to Gitea's registry (Development Plan
+# A published release builds the agent's image and pushes it to Gitea's
-# M1.5, X4): gitea.dongar.net/dongar/nisse-dev:<tag> and :latest. The token
+# registry (Development Plan M1.5, X4): gitea.dongar.net/dev/nisse-dev
-# is a secret (REGISTRY_TOKEN: write:package for REGISTRY_USER); the
+# tagged with the version (the release tag without its v) and latest. Cut a
-# account is a setting, so it's here.
+# release in Gitea (Releases → New release, tag vX.Y.Z); a tag pushed for any
+# other reason builds nothing.
+#
+# The registry needs a personal access token with write:package
+# (REGISTRY_TOKEN, an organization secret of `dev`, belonging to the
+# registry-push bot), never the built-in job token: Gitea's registry
+# resolves an Actions job token to an empty package scope (go-gitea/gitea
+# #23642 and others). Packages belong to the repository's owner, which is
+# why this repository is the `dev` organization's: the bot writes packages
+# through the org's registry-push team. Never on pull_request: registry
+# credentials stay off that trigger.
 on:
-  push:
+  release:
-    tags: ["v*"]
+    types: [published]
 
 env:
   REGISTRY: gitea.dongar.net
-  REGISTRY_USER: dongar
+  REGISTRY_USER: registry-push  # the token's account
-  IMAGE: gitea.dongar.net/dongar/nisse-dev
+  IMAGE: gitea.dongar.net/dev/nisse-dev
 
 jobs:
-  image:
+  build-and-push:
     runs-on: ubuntu-latest
+    permissions:
+      contents: read
     steps:
-      - uses: actions/checkout@v4
+      - name: Checkout
+        uses: actions/checkout@v4
 
-      - name: Build
+      - name: Derive the image version from the release's tag
-        run: |
+        id: vars
-          docker build --target agent \
-            --label org.opencontainers.image.revision="${{ github.sha }}" \
-            --label org.opencontainers.image.version="${{ github.ref_name }}" \
-            -t "$IMAGE:${{ github.ref_name }}" -t "$IMAGE:latest" .
-
-      - name: Push
         env:
-          REGISTRY_TOKEN: ${{ secrets.REGISTRY_TOKEN }}
+          TAG: ${{ github.event.release.tag_name }}
-        run: |
+        run: echo "version=${TAG#v}" >> "$GITHUB_OUTPUT"
-          printf '%s' "$REGISTRY_TOKEN" | docker login "$REGISTRY" -u "$REGISTRY_USER" --password-stdin
+
-          docker push "$IMAGE:${{ github.ref_name }}"
+      - name: Log in to Gitea's container registry
-          docker push "$IMAGE:latest"
+        uses: docker/login-action@v3
-          docker logout "$REGISTRY"
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ env.REGISTRY_USER }}
+          password: ${{ secrets.REGISTRY_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and push the agent
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          target: agent
+          push: true
+          build-args: |
+            VERSION=${{ steps.vars.outputs.version }}
+          labels: |
+            org.opencontainers.image.version=${{ steps.vars.outputs.version }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          tags: |
+            ${{ env.IMAGE }}:${{ steps.vars.outputs.version }}
+            ${{ env.IMAGE }}:latest

Dockerfile

@@ -24,6 +24,8 @@ COPY nisse-ops nisse-ops
 RUN uv sync --frozen --no-dev --package nisse-dev --no-editable
 
 FROM python:3.12-slim AS runtime
+ARG VERSION=dev
+LABEL org.opencontainers.image.title=nisse-dev org.opencontainers.image.version=$VERSION
 RUN apt-get update \
     && apt-get install -y --no-install-recommends openssh-client \
     && rm -rf /var/lib/apt/lists/* \

docs/Development-Plan.md

@@ -280,9 +280,9 @@ accepted once a tag's image passes the e2e suite.
 - The image runs as its own user; state in `/var/lib/nisse-dev`, configuration
   mounted at `/etc/nisse-dev/nisse-dev.yaml`, secrets as environment variables,
   keys mounted read-only. Its `test` stage carries the tests over the installed
-  package, for `scripts/e2e.sh --image REF`. CI builds both on every PR;
+  package, for `scripts/e2e.sh --image REF`. Images are built only on release:
-  `release.yml` pushes `gitea.dongar.net/dongar/nisse-dev:<tag>` and `:latest`
+  `release.yml` pushes `gitea.dongar.net/dev/nisse-dev:<version>` and `:latest`
-  (`REGISTRY_TOKEN`).
+  on a published release (`REGISTRY_TOKEN`, the `dev` organization's).
 - The live bring-up's switch to the image waits for the next live test (it needs
   its own test repository too); `bringup.sh plan` shows what it will run.
 

nisse-dev/src/nisse_dev/llm.py

@@ -39,6 +39,13 @@ class LLMError(RuntimeError):
 RATE_LIMIT_STATUS = 429
 RATE_LIMIT_RETRY_S = 20.0
 
+# The proxy, or what's behind it, briefly gone: a 502/503/504 or a dropped
+# connection. Outages delay, never fail (L5): waited out with backoff before
+# the work is stopped. Not a 500 (it may be the request) and not a timeout
+# (a hung model has had its request_timeout_s already).
+OUTAGE_STATUSES = frozenset({502, 503, 504})
+OUTAGE_BACKOFF_S = (5.0, 15.0, 45.0)
+
 # Generous by default. Output is billed by what is produced, not by the
 # ceiling, and a ceiling too low does not fail loudly — it truncates a tool
 # call mid-argument and hands back something that parses. Measured: Opus
@@ -381,20 +388,39 @@ class AnthropicPassthroughClient:
 def _post_with_rate_limit_retry(
     http: httpx.Client, path: str, body: dict[str, Any], sleep: Any
 ) -> tuple[httpx.Response, float]:
-    """The response, and how long was spent waiting out a rate limit."""
+    """The response, and how long was spent waiting: out a rate limit (once)
+    or an outage (with backoff), not on the model."""
     waited = 0.0
-    for attempt in (1, 2):
+    rate_limited = False
+    outages = iter(OUTAGE_BACKOFF_S)
+    while True:
         try:
             response = http.post(path, json=body)
+        except httpx.TimeoutException as exc:
+            raise UpstreamUnavailable(f"LLM request failed: {exc}") from exc
         except httpx.HTTPError as exc:
             # Could not reach the proxy at all: nothing about the request was
             # wrong, so callers holding work in progress should keep it.
+            pause = next(outages, None)
+            if pause is None:
                 raise UpstreamUnavailable(f"LLM request failed: {exc}") from exc
-        if response.status_code != RATE_LIMIT_STATUS or attempt == 2:
+            sleep(pause)
-            return response, waited
+            waited += pause
+            continue
+        if response.status_code == RATE_LIMIT_STATUS and not rate_limited:
+            rate_limited = True
             sleep(RATE_LIMIT_RETRY_S)
             waited += RATE_LIMIT_RETRY_S
-    raise AssertionError("unreachable")
+            continue
+        if response.status_code in OUTAGE_STATUSES and not _looks_like_budget_refusal(
+            response.text
+        ):
+            pause = next(outages, None)
+            if pause is not None:
+                sleep(pause)
+                waited += pause
+                continue
+        return response, waited
 
 
 # Model families we have a native renderer adapter for. L9: delegations take

nisse-dev/tests/test_agent_build.py

@@ -143,6 +143,9 @@ class FakeTransport:
 def build_offline(tmp_path: Path, origin: Path, monkeypatch: Any) -> Any:
     """Assemble the real graph with only the network boundaries faked."""
     config = config_for(tmp_path)
+    # The proxy here doesn't exist: a call to it fails at once rather than
+    # waiting out an outage's backoff (L5).
+    monkeypatch.setattr("nisse_dev.llm.OUTAGE_BACKOFF_S", ())
 
     monkeypatch.setattr(
         "nisse_dev.agent.GiteaClient.whoami", lambda self: "nisse-dev-testing"

nisse-dev/tests/test_llm.py

@@ -14,6 +14,7 @@ from nisse_dev.canonical import (
 from nisse_dev.envfile import load_env_file
 from nisse_dev.llm import (
     DEFAULT_MAX_OUTPUT_TOKENS,
+    OUTAGE_BACKOFF_S,
     RATE_LIMIT_RETRY_S,
     AnthropicPassthroughClient,
     BudgetExhausted,
@@ -432,3 +433,81 @@ def test_the_answering_model_is_recorded() -> None:
     )
     client.complete(EVENTS, tools=None, model="local-fast")
     assert client.last_model == "gpt-test-1"
+
+
+# --- outages delay, never fail (L5) --------------------------------------------
+
+
+def test_a_gateway_error_is_waited_out_with_backoff() -> None:
+    responses = [
+        httpx.Response(502, text="bad gateway"),
+        httpx.Response(503, text="unavailable"),
+        completion({"content": "back"}),
+    ]
+    waited: list[float] = []
+    client = make_client(lambda request: responses.pop(0), sleep=waited.append)
+    [event] = client.complete(EVENTS, tools=None, model="local/qwen")
+    assert isinstance(event, AssistantText) and event.text == "back"
+    assert waited == list(OUTAGE_BACKOFF_S[:2])
+    # Waiting isn't the model's time.
+    assert client.last_rate_limit_wait_s == sum(OUTAGE_BACKOFF_S[:2])
+
+
+def test_an_outage_that_outlasts_the_backoff_is_surfaced() -> None:
+    attempts: list[int] = []
+    waited: list[float] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        attempts.append(1)
+        return httpx.Response(502, text="bad gateway")
+
+    client = make_client(handler, sleep=waited.append)
+    with pytest.raises(UpstreamUnavailable, match="502"):
+        client.complete(EVENTS, tools=None, model="local/qwen")
+    assert len(attempts) == len(OUTAGE_BACKOFF_S) + 1
+    assert waited == list(OUTAGE_BACKOFF_S)
+
+
+def test_a_dropped_connection_is_retried() -> None:
+    calls: list[int] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(1)
+        if len(calls) == 1:
+            raise httpx.ConnectError("refused")
+        return completion({"content": "ok"})
+
+    [event] = make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
+    assert isinstance(event, AssistantText) and len(calls) == 2
+
+
+def test_a_500_or_a_timeout_is_not_retried() -> None:
+    # A 500 may be the request's fault; a timeout already had its full
+    # request_timeout_s.
+    for failure in (
+        lambda request: httpx.Response(500, text="boom"),
+        lambda request: (_ for _ in ()).throw(httpx.ReadTimeout("slow")),
+    ):
+        calls: list[int] = []
+
+        def handler(
+            request: httpx.Request, failure=failure, calls=calls
+        ) -> httpx.Response:
+            calls.append(1)
+            return failure(request)
+
+        with pytest.raises(LLMError):
+            make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
+        assert len(calls) == 1
+
+
+def test_a_budget_refusal_is_not_waited_on() -> None:
+    calls: list[int] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        calls.append(1)
+        return httpx.Response(503, text="Budget has been exceeded! Current cost: 5.1")
+
+    with pytest.raises(BudgetExhausted):
+        make_client(handler).complete(EVENTS, tools=None, model="local/qwen")
+    assert len(calls) == 1