Stop CI running two suites at once, and retry rate limits
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m5s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 1m5s
CI failed on the delegation branch with a 429 from the proxy: token limit 200000, 17399 remaining. Not a defect — the live delegation tests are token-hungry, and every push to a PR branch fired two identical suites concurrently, since the push and pull_request events both matched. - Push now only triggers on main. A PR branch is already covered by the pull_request event, so this halves both the wall time and the tokens spent per branch - A 429 is retried once before being surfaced. Losing a ten-turn delegation to a transient limit wastes far more than waiting out the window, and retrying forever would hide a real ceiling. The sleep is injectable, so the existing 429 test no longer waits 20 seconds for real Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
1 parent
53ed11f9c4
commit
af9bdaac8d
3 files changed
+67
-11
No files matched your search
@@ -11,16 +11,17 @@ from nisse_dev.canonical import (
|
||||
UserMessage,
|
||||
)
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.llm import LiteLLMClient, LLMError, RoleMap
|
||||
from nisse_dev.llm import RATE_LIMIT_RETRY_S, LiteLLMClient, LLMError, RoleMap
|
||||
|
||||
EVENTS = [StoredEvent(id=1, ts="t", event=UserMessage(text="hi", source="m:1"))]
|
||||
|
||||
|
||||
def make_client(handler) -> LiteLLMClient:
|
||||
def make_client(handler, sleep=lambda _s: None) -> LiteLLMClient:
|
||||
return LiteLLMClient(
|
||||
"http://litellm.test/v1",
|
||||
"sk-test",
|
||||
transport=httpx.MockTransport(handler),
|
||||
sleep=sleep, # never wait in unit tests
|
||||
)
|
||||
|
||||
|
||||
@@ -109,9 +110,34 @@ def test_non_object_tool_arguments_raise_llm_error() -> None:
|
||||
|
||||
|
||||
def test_http_error_status_raises_llm_error() -> None:
|
||||
client = make_client(lambda request: httpx.Response(429, text="rate limited"))
|
||||
client = make_client(lambda request: httpx.Response(500, text="boom"))
|
||||
with pytest.raises(LLMError, match="500"):
|
||||
client.complete(EVENTS, tools=None, model="local/qwen")
|
||||
|
||||
|
||||
def test_rate_limit_is_retried_once_then_surfaced() -> None:
|
||||
# A 429 is transient. Losing a long delegation to one costs far more
|
||||
# than waiting out the window, but retrying forever would hide a real
|
||||
# ceiling, so it is retried exactly once.
|
||||
attempts: list[int] = []
|
||||
waited: list[float] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
attempts.append(1)
|
||||
return httpx.Response(429, text="rate limited")
|
||||
|
||||
client = make_client(handler, sleep=waited.append)
|
||||
with pytest.raises(LLMError, match="429"):
|
||||
client.complete(EVENTS, tools=None, model="local/qwen")
|
||||
assert len(attempts) == 2
|
||||
assert waited == [RATE_LIMIT_RETRY_S]
|
||||
|
||||
|
||||
def test_rate_limit_that_clears_succeeds() -> None:
|
||||
responses = [httpx.Response(429, text="slow down"), completion({"content": "hi"})]
|
||||
client = make_client(lambda request: responses.pop(0), sleep=lambda _s: None)
|
||||
[event] = client.complete(EVENTS, tools=None, model="local/qwen")
|
||||
assert isinstance(event, AssistantText)
|
||||
|
||||
|
||||
def test_connection_failure_raises_llm_error() -> None:
|
||||
|
||||
Reference in new issue
Block a user