A model's safety-filter refusal is an error that says so, not an empty reply #165
No files matched your search
@@ -32,7 +32,7 @@ from nisse_core.log.canonical import (
|
||||
StoredEvent,
|
||||
UserMessage,
|
||||
)
|
||||
from nisse_core.models.llm import LLMClient, LLMError, RoleMap
|
||||
from nisse_core.models.llm import LLMClient, LLMError, ModelRefused, RoleMap
|
||||
from nisse_core.state.store import Transcript
|
||||
|
||||
EXPERT_ROLE = "expert"
|
||||
@@ -168,6 +168,9 @@ class ExpertDesk:
|
||||
model=model,
|
||||
system=system,
|
||||
)
|
||||
except ModelRefused as exc:
|
||||
# Not retried: the same question is declined again.
|
||||
return None, f"{exc}. Ask it differently, or work from the repository"
|
||||
except LLMError as exc:
|
||||
return None, str(exc)
|
||||
text = "\n".join(
|
||||
|
||||
@@ -102,6 +102,24 @@ class TruncatedResponse(LLMError):
|
||||
"""
|
||||
|
||||
|
||||
class ModelRefused(LLMError):
|
||||
"""The provider's safety filter declined to answer.
|
||||
|
||||
It comes back as a successful reply with no text (``finish_reason:
|
||||
content_filter`` on the chat route, ``stop_reason: refusal`` on the
|
||||
passthrough), so unchecked it reads as an empty answer. It can be a false
|
||||
positive: Opus declined, every time, a question about which repositories
|
||||
a Gitea token can list. Asking the same thing again gets the same answer.
|
||||
"""
|
||||
|
||||
|
||||
def _refused(model: str) -> ModelRefused:
|
||||
return ModelRefused(
|
||||
f"{model} declined to answer: its provider's safety filter stopped the "
|
||||
"reply. That says nothing about the subject itself"
|
||||
)
|
||||
|
||||
|
||||
class LLMClient(Protocol):
|
||||
def complete(
|
||||
self,
|
||||
@@ -216,6 +234,8 @@ class LiteLLMClient:
|
||||
f"{requested_model} hit its output limit mid-answer; raise "
|
||||
"llm.max_output_tokens, or the model's max_output_tokens in LiteLLM"
|
||||
)
|
||||
if choice.get("finish_reason") == "content_filter":
|
||||
raise _refused(requested_model)
|
||||
produced_by = data.get("model") or requested_model
|
||||
# What actually answered, which is not always what was asked for: a
|
||||
# proxy fallback answers under another model's name.
|
||||
@@ -338,6 +358,8 @@ class AnthropicPassthroughClient:
|
||||
f"{requested_model} hit its output limit mid-answer; raise "
|
||||
"llm.max_output_tokens, or the model's max_output_tokens in LiteLLM"
|
||||
)
|
||||
if data.get("stop_reason") == "refusal":
|
||||
raise _refused(requested_model)
|
||||
content = data.get("content")
|
||||
if not isinstance(content, list):
|
||||
raise LLMError(f"malformed passthrough response: {str(data)[:300]}")
|
||||
|
||||
@@ -18,7 +18,7 @@ from nisse_core.loop.expert import (
|
||||
ExpertDesk,
|
||||
parse_answer,
|
||||
)
|
||||
from nisse_core.models.llm import LLMError, RoleMap
|
||||
from nisse_core.models.llm import LLMError, RoleMap, _refused
|
||||
from nisse_core.oplog import OpLog
|
||||
from nisse_core.state.store import Transcript
|
||||
from nisse_dev.state import StateStore
|
||||
@@ -142,6 +142,33 @@ def test_unparseable_reply_is_retried_once_then_raises(store: StateStore) -> Non
|
||||
assert "not a single valid JSON object" in client.calls[1]["system"]
|
||||
|
||||
|
||||
class DecliningClient(ScriptedClient):
|
||||
"""The provider's safety filter, declining every time."""
|
||||
|
||||
def complete(self, events: Any, tools: Any, model: str, system: Any = None) -> Any:
|
||||
self.calls.append({"model": model, "system": system})
|
||||
raise _refused(model) # the clients' own wording
|
||||
|
||||
|
||||
def test_a_declined_question_is_not_asked_again_and_says_why(
|
||||
store: StateStore,
|
||||
) -> None:
|
||||
client = DecliningClient([])
|
||||
with pytest.raises(LLMError) as raised:
|
||||
desk(client, store).ask("which repositories can a token list?")
|
||||
assert len(client.calls) == 1 # the same question is declined again
|
||||
assert "Ask it differently, or work from the repository" in str(raised.value)
|
||||
[completed] = [
|
||||
e.event for e in log(store).read() if isinstance(e.event, DelegationCompleted)
|
||||
]
|
||||
assert completed.exit_status == "failed"
|
||||
assert completed.handback == (
|
||||
|
|
||||
"claude-haiku-4-5 declined to answer: its provider's safety filter stopped the "
|
||||
"reply. That says nothing about the subject itself. Ask it "
|
||||
"differently, or work from the repository"
|
||||
)
|
||||
|
||||
|
||||
def test_retry_succeeds_after_a_bad_first_reply(store: StateStore) -> None:
|
||||
client = ScriptedClient(["waffle", GOOD])
|
||||
assert desk(client, store).ask("q").confidence == "high"
|
||||
|
||||
@@ -20,6 +20,7 @@ from nisse_core.models.llm import (
|
||||
BudgetExhausted,
|
||||
LiteLLMClient,
|
||||
LLMError,
|
||||
ModelRefused,
|
||||
RoleMap,
|
||||
TruncatedResponse,
|
||||
UpstreamUnavailable,
|
||||
@@ -295,6 +296,51 @@ def test_anthropic_route_raises_on_truncation() -> None:
|
||||
client.complete(EVENTS, tools=None, model="claude-opus-5")
|
||||
|
||||
|
||||
# --- the provider's safety filter -------------------------------------------
|
||||
|
||||
|
||||
def test_openai_route_raises_when_the_filter_declines() -> None:
|
||||
# As LiteLLM reports Opus's refusal (seen live, #164 follow-up): a 200
|
||||
# with no text, which used to read as an empty answer.
|
||||
declined = httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "claude-opus-5-5",
|
||||
"choices": [
|
||||
{
|
||||
"finish_reason": "content_filter",
|
||||
"message": {"role": "assistant", "content": None},
|
||||
}
|
||||
],
|
||||
"usage": {"prompt_tokens": 612, "completion_tokens": 0},
|
||||
},
|
||||
)
|
||||
client = make_client(lambda request: declined)
|
||||
with pytest.raises(ModelRefused, match="safety filter"):
|
||||
client.complete(EVENTS, tools=None, model="claude-opus-5-5")
|
||||
|
||||
|
||||
def test_anthropic_route_raises_when_the_filter_declines() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"model": "claude-opus-5-5", "stop_reason": "refusal", "content": []},
|
||||
)
|
||||
|
||||
client = AnthropicPassthroughClient(
|
||||
"https://litellm.test",
|
||||
"tok",
|
||||
transport=httpx.MockTransport(handler),
|
||||
sleep=lambda _s: None,
|
||||
)
|
||||
with pytest.raises(ModelRefused, match="declined to answer"):
|
||||
client.complete(EVENTS, tools=None, model="claude-opus-5-5")
|
||||
|
||||
|
||||
def test_a_refusal_is_an_llm_error_so_callers_already_handle_it() -> None:
|
||||
assert issubclass(ModelRefused, LLMError)
|
||||
|
||||
|
||||
def test_a_complete_answer_is_not_mistaken_for_truncation() -> None:
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(
|
||||
|
||||
Reference in new issue
Block a user
Agreed, done in
369b6ed.DecliningClientnow raises_refused(model)fromllm.py, and the test compares the whole handback, so it fails if the clients' wording changes (checked: dropping "safety" from_refusedfails it).Verified at head and confirmed addressed:
DecliningClientraises_refused(model)(test_expert.py:150) and the test asserts the full handback string (165-169), so a wording change in either_refused()or the desk's wrapping now fails the test — exactly what was asked. Importing the private_refusedin a test is consistent with the repo's existing pattern (_HELPER,_parse_projects), so no further nit. Point closed.