Record rate-limit waits in benchmark runs, and leave them out of time
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m17s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m17s
The client waits RATE_LIMIT_RETRY_S and retries once on a 429, silently. A benchmark run that waited looked slower than the model is: on 2026-09-27, fast models under the unit-test key's 200k tokens/minute limit had 20-45 s runs that were probably waits. Both clients now report last_rate_limit_wait_s after every call, even one that then fails. The benchmark sums it per run into rate_limit_wait_s, the report's time column leaves it out, and a new 429 column counts the runs that waited. Results stored before this load unchanged. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
a716d5ec8d
commit
4e45dfe06a
6 files changed
+116
-9
No files matched your search
@@ -142,6 +142,9 @@ def test_rate_limit_is_retried_once_then_surfaced() -> None:
|
||||
client.complete(EVENTS, tools=None, model="local/qwen")
|
||||
assert len(attempts) == 2
|
||||
assert waited == [RATE_LIMIT_RETRY_S]
|
||||
# Reported even though the call failed, so a caller can leave the wait
|
||||
# out of how long the model took.
|
||||
assert client.last_rate_limit_wait_s == RATE_LIMIT_RETRY_S
|
||||
|
||||
|
||||
def test_rate_limit_that_clears_succeeds() -> None:
|
||||
@@ -149,6 +152,19 @@ def test_rate_limit_that_clears_succeeds() -> None:
|
||||
client = make_client(lambda request: responses.pop(0), sleep=lambda _s: None)
|
||||
[event] = client.complete(EVENTS, tools=None, model="local/qwen")
|
||||
assert isinstance(event, AssistantText)
|
||||
assert client.last_rate_limit_wait_s == RATE_LIMIT_RETRY_S
|
||||
|
||||
|
||||
def test_the_rate_limit_wait_is_per_call() -> None:
|
||||
responses = [
|
||||
httpx.Response(429, text="slow down"),
|
||||
completion({"content": "one"}),
|
||||
completion({"content": "two"}),
|
||||
]
|
||||
client = make_client(lambda request: responses.pop(0), sleep=lambda _s: None)
|
||||
client.complete(EVENTS, tools=None, model="local/qwen")
|
||||
client.complete(EVENTS, tools=None, model="local/qwen")
|
||||
assert client.last_rate_limit_wait_s == 0.0
|
||||
|
||||
|
||||
def test_connection_failure_raises_llm_error() -> None:
|
||||
|
||||
Reference in new issue
Block a user