Record rate-limit waits in benchmark runs, and leave them out of time
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m17s

The client waits RATE_LIMIT_RETRY_S and retries once on a 429, silently.
A benchmark run that waited looked slower than the model is: on
2026-09-27, fast models under the unit-test key's 200k tokens/minute
limit had 20-45 s runs that were probably waits.

Both clients now report last_rate_limit_wait_s after every call, even
one that then fails. The benchmark sums it per run into
rate_limit_wait_s, the report's time column leaves it out, and a new
429 column counts the runs that waited. Results stored before this
load unchanged.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-09-27 20:57:15 +00:00
1 parent a716d5ec8d
commit 4e45dfe06a
6 files changed
+116 -9

No files matched your search

+16
View File
@@ -142,6 +142,9 @@ def test_rate_limit_is_retried_once_then_surfaced() -> None:
client.complete(EVENTS, tools=None, model="local/qwen")
assert len(attempts) == 2
assert waited == [RATE_LIMIT_RETRY_S]
# Reported even though the call failed, so a caller can leave the wait
# out of how long the model took.
assert client.last_rate_limit_wait_s == RATE_LIMIT_RETRY_S
def test_rate_limit_that_clears_succeeds() -> None:
@@ -149,6 +152,19 @@ def test_rate_limit_that_clears_succeeds() -> None:
client = make_client(lambda request: responses.pop(0), sleep=lambda _s: None)
[event] = client.complete(EVENTS, tools=None, model="local/qwen")
assert isinstance(event, AssistantText)
assert client.last_rate_limit_wait_s == RATE_LIMIT_RETRY_S
def test_the_rate_limit_wait_is_per_call() -> None:
responses = [
httpx.Response(429, text="slow down"),
completion({"content": "one"}),
completion({"content": "two"}),
]
client = make_client(lambda request: responses.pop(0), sleep=lambda _s: None)
client.complete(EVENTS, tools=None, model="local/qwen")
client.complete(EVENTS, tools=None, model="local/qwen")
assert client.last_rate_limit_wait_s == 0.0
def test_connection_failure_raises_llm_error() -> None: