A model's safety-filter refusal is an error that says so, not an empty reply
Opus's provider filter declined one of nisse-review's ask_expert questions (which repositories a Gitea token can list) every time. It came back as a 200 with no text, and the expert reported "unparseable reply: empty reply". Both LLM clients now raise ModelRefused for finish_reason content_filter (chat route) and stop_reason refusal (passthrough). The expert's handback says what happened and what to do instead. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
1 parent
098d42ff2c
commit
9d2992c20a
4 files changed
+96
-2
No files matched your search
@@ -18,7 +18,7 @@ from nisse_core.loop.expert import (
|
||||
ExpertDesk,
|
||||
parse_answer,
|
||||
)
|
||||
from nisse_core.models.llm import LLMError, RoleMap
|
||||
from nisse_core.models.llm import LLMError, ModelRefused, RoleMap
|
||||
from nisse_core.oplog import OpLog
|
||||
from nisse_core.state.store import Transcript
|
||||
from nisse_dev.state import StateStore
|
||||
@@ -142,6 +142,29 @@ def test_unparseable_reply_is_retried_once_then_raises(store: StateStore) -> Non
|
||||
assert "not a single valid JSON object" in client.calls[1]["system"]
|
||||
|
||||
|
||||
class DecliningClient(ScriptedClient):
|
||||
"""The provider's safety filter, declining every time."""
|
||||
|
||||
def complete(self, events: Any, tools: Any, model: str, system: Any = None) -> Any:
|
||||
self.calls.append({"model": model, "system": system})
|
||||
raise ModelRefused(f"{model} declined to answer: its provider's safety filter")
|
||||
|
||||
|
||||
def test_a_declined_question_is_not_asked_again_and_says_why(
|
||||
store: StateStore,
|
||||
) -> None:
|
||||
client = DecliningClient([])
|
||||
with pytest.raises(LLMError) as raised:
|
||||
desk(client, store).ask("which repositories can a token list?")
|
||||
assert len(client.calls) == 1 # the same question is declined again
|
||||
assert "Ask it differently, or work from the repository" in str(raised.value)
|
||||
[completed] = [
|
||||
e.event for e in log(store).read() if isinstance(e.event, DelegationCompleted)
|
||||
]
|
||||
assert completed.exit_status == "failed"
|
||||
assert "safety filter" in completed.handback
|
||||
|
||||
|
||||
def test_retry_succeeds_after_a_bad_first_reply(store: StateStore) -> None:
|
||||
client = ScriptedClient(["waffle", GOOD])
|
||||
assert desk(client, store).ask("q").confidence == "high"
|
||||
|
||||
@@ -20,6 +20,7 @@ from nisse_core.models.llm import (
|
||||
BudgetExhausted,
|
||||
LiteLLMClient,
|
||||
LLMError,
|
||||
ModelRefused,
|
||||
RoleMap,
|
||||
TruncatedResponse,
|
||||
UpstreamUnavailable,
|
||||
@@ -295,6 +296,51 @@ def test_anthropic_route_raises_on_truncation() -> None:
|
||||
client.complete(EVENTS, tools=None, model="claude-opus-5")
|
||||
|
||||
|
||||
# --- the provider's safety filter -------------------------------------------
|
||||
|
||||
|
||||
def test_openai_route_raises_when_the_filter_declines() -> None:
|
||||
# As LiteLLM reports Opus's refusal (seen live, #164 follow-up): a 200
|
||||
# with no text, which used to read as an empty answer.
|
||||
declined = httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "claude-opus-5-5",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "content_filter",
|
||||
"message": {"role": "assistant", "content": None},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 612, "completion_tokens": 0},
|
||||
},
|
||||
)
|
||||
client = make_client(lambda request: declined)
|
||||
with pytest.raises(ModelRefused, match="safety filter"):
|
||||
client.complete(EVENTS, tools=None, model="claude-opus-5-5")
|
||||
|
||||
|
||||
def test_anthropic_route_raises_when_the_filter_declines() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"model": "claude-opus-5-5", "stop_reason": "refusal", "content": []},
|
||||
)
|
||||
|
||||
client = AnthropicPassthroughClient(
|
||||
"https://litellm.test",
|
||||
"tok",
|
||||
transport=httpx.MockTransport(handler),
|
||||
sleep=lambda _s: None,
|
||||
)
|
||||
with pytest.raises(ModelRefused, match="declined to answer"):
|
||||
client.complete(EVENTS, tools=None, model="claude-opus-5-5")
|
||||
|
||||
|
||||
def test_a_refusal_is_an_llm_error_so_callers_already_handle_it() -> None:
|
||||
assert issubclass(ModelRefused, LLMError)
|
||||
|
||||
|
||||
def test_a_complete_answer_is_not_mistaken_for_truncation() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
|
||||
Reference in new issue
Block a user