Record rate-limit waits in benchmark runs, and leave them out of time

The client waits RATE_LIMIT_RETRY_S and retries once on a 429, silently.
A benchmark run that waited looked slower than the model is: on
2026-09-27, fast models under the unit-test key's 200k tokens/minute
limit had 20-45 s runs that were probably waits.

Both clients now report last_rate_limit_wait_s after every call, even
one that then fails. The benchmark sums it per run into
rate_limit_wait_s, the report's time column leaves it out, and a new
429 column counts the runs that waited. Results stored before this
load unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

nisse-dev/src/nisse_dev/llm.py

@@ -144,6 +144,9 @@ class LiteLLMClient:
         self.last_usage: dict[str, int] = {}
         self.last_model: str | None = None
         self.last_cost: float = 0.0
+        # Set even when the call then fails: time spent waiting out a rate
+        # limit is the proxy's, not the model's (the benchmark subtracts it).
+        self.last_rate_limit_wait_s: float = 0.0
 
     def complete(
         self,
@@ -164,7 +167,8 @@ class LiteLLMClient:
         # nothing to check against. A caller that knows the model's limits
         # (the context budget) passes a tighter one.
         body["max_tokens"] = max_tokens or self._max_tokens
-        response = _post_with_rate_limit_retry(
+        self.last_rate_limit_wait_s = 0.0  # not the last call's, if this one raises
+        response, self.last_rate_limit_wait_s = _post_with_rate_limit_retry(
             self._http, "/chat/completions", body, self._sleep
         )
         if response.status_code != 200:
@@ -267,6 +271,9 @@ class AnthropicPassthroughClient:
         self.last_usage: dict[str, int] = {}
         self.last_model: str | None = None
         self.last_cost: float = 0.0
+        # Set even when the call then fails: time spent waiting out a rate
+        # limit is the proxy's, not the model's (the benchmark subtracts it).
+        self.last_rate_limit_wait_s: float = 0.0
 
     def complete(
         self,
@@ -287,7 +294,8 @@ class AnthropicPassthroughClient:
         )
         if require_tool_call and tools:
             body["tool_choice"] = {"type": "any"}  # Anthropic's name for it
-        response = _post_with_rate_limit_retry(
+        self.last_rate_limit_wait_s = 0.0  # not the last call's, if this one raises
+        response, self.last_rate_limit_wait_s = _post_with_rate_limit_retry(
             self._http, "/v1/messages", body, self._sleep
         )
         if response.status_code != 200:
@@ -347,7 +355,9 @@ class AnthropicPassthroughClient:
 
 def _post_with_rate_limit_retry(
     http: httpx.Client, path: str, body: dict[str, Any], sleep: Any
-) -> httpx.Response:
+) -> tuple[httpx.Response, float]:
+    """The response, and how long was spent waiting out a rate limit."""
+    waited = 0.0
     for attempt in (1, 2):
         try:
             response = http.post(path, json=body)
@@ -356,8 +366,9 @@ def _post_with_rate_limit_retry(
             # wrong, so callers holding work in progress should keep it.
             raise UpstreamUnavailable(f"LLM request failed: {exc}") from exc
         if response.status_code != RATE_LIMIT_STATUS or attempt == 2:
-            return response
+            return response, waited
         sleep(RATE_LIMIT_RETRY_S)
+        waited += RATE_LIMIT_RETRY_S
     raise AssertionError("unreachable")