Record rate-limit waits in benchmark runs, and leave them out of time

The client waits RATE_LIMIT_RETRY_S and retries once on a 429, silently.
A benchmark run that waited looked slower than the model is: on
2026-09-27, fast models under the unit-test key's 200k tokens/minute
limit had 20-45 s runs that were probably waits.

Both clients now report last_rate_limit_wait_s after every call, even
one that then fails. The benchmark sums it per run into
rate_limit_wait_s, the report's time column leaves it out, and a new
429 column counts the runs that waited. Results stored before this
load unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

nisse-dev/tests/test_bench.py

@@ -263,3 +263,57 @@ def test_metering_counts_prose_and_cost() -> None:
     meter.complete([], None, "local-fast")
     meter.complete([], None, "local-fast")
     assert (meter.calls, meter.prose_turns, meter.cost) == (2, 2, 1.0)
+
+
+def test_metering_counts_rate_limit_waits_even_on_failure() -> None:
+    class Inner:
+        def __init__(self) -> None:
+            self.last_usage = {"input": 10, "output": 2}
+            self.last_cost = 0.0
+            self.last_model = "local-fast"
+            self.last_rate_limit_wait_s = 0.0
+            self.calls = 0
+
+        def complete(self, events, tools, model, system=None):
+            self.calls += 1
+            self.last_rate_limit_wait_s = 20.0
+            if self.calls == 2:
+                raise RuntimeError("upstream returned 429")
+            return [AssistantText(text="hi", model=model)]
+
+    meter = MeteringClient(Inner(), FACTS)
+    meter.complete([], None, "local-fast")
+    with pytest.raises(RuntimeError):
+        meter.complete([], None, "local-fast")
+    assert meter.rate_limit_wait_s == 40.0
+
+
+def test_a_run_records_its_rate_limit_waits(tmp_path: Path) -> None:
+    [task] = load_tasks(["write-function"])
+    client = Scripted(list(SOLVE_SLUGIFY))
+    client.last_rate_limit_wait_s = 20.0  # every call waited once
+    record = run_task(task, FACTS, client, Tuning())
+    assert record.rate_limit_wait_s == 60.0
+
+
+def test_the_report_leaves_rate_limit_waits_out_of_time() -> None:
+    current = {t.name: t.content_hash for t in TASKS}
+    rows = summarise(
+        [
+            record_json(wall_s=50.0, rate_limit_wait_s=40.0),
+            record_json(wall_s=10.0, rate_limit_wait_s=0.0),
+        ],
+        current,
+    )
+    [row] = rows
+    assert row.mean_wall_s == 10.0
+    assert row.rate_limited == 1
+    assert "429 = runs that waited" in render(rows)
+
+
+def test_results_stored_before_waits_were_recorded_still_load() -> None:
+    current = {t.name: t.content_hash for t in TASKS}
+    old = record_json(wall_s=12.0)
+    del old["rate_limit_wait_s"]
+    [row] = summarise([old], current)
+    assert (row.mean_wall_s, row.rate_limited) == (12.0, 0)

nisse-dev/tests/test_llm.py

@@ -142,6 +142,9 @@ def test_rate_limit_is_retried_once_then_surfaced() -> None:
         client.complete(EVENTS, tools=None, model="local/qwen")
     assert len(attempts) == 2
     assert waited == [RATE_LIMIT_RETRY_S]
+    # Reported even though the call failed, so a caller can leave the wait
+    # out of how long the model took.
+    assert client.last_rate_limit_wait_s == RATE_LIMIT_RETRY_S
 
 
 def test_rate_limit_that_clears_succeeds() -> None:
@@ -149,6 +152,19 @@ def test_rate_limit_that_clears_succeeds() -> None:
     client = make_client(lambda request: responses.pop(0), sleep=lambda _s: None)
     [event] = client.complete(EVENTS, tools=None, model="local/qwen")
     assert isinstance(event, AssistantText)
+    assert client.last_rate_limit_wait_s == RATE_LIMIT_RETRY_S
+
+
+def test_the_rate_limit_wait_is_per_call() -> None:
+    responses = [
+        httpx.Response(429, text="slow down"),
+        completion({"content": "one"}),
+        completion({"content": "two"}),
+    ]
+    client = make_client(lambda request: responses.pop(0), sleep=lambda _s: None)
+    client.complete(EVENTS, tools=None, model="local/qwen")
+    client.complete(EVENTS, tools=None, model="local/qwen")
+    assert client.last_rate_limit_wait_s == 0.0
 
 
 def test_connection_failure_raises_llm_error() -> None: