diff --git a/nisse-core/src/nisse_core/loop/expert.py b/nisse-core/src/nisse_core/loop/expert.py index b5784a5..73cac3a 100644 --- a/nisse-core/src/nisse_core/loop/expert.py +++ b/nisse-core/src/nisse_core/loop/expert.py @@ -32,7 +32,7 @@ from nisse_core.log.canonical import ( StoredEvent, UserMessage, ) -from nisse_core.models.llm import LLMClient, LLMError, RoleMap +from nisse_core.models.llm import LLMClient, LLMError, ModelRefused, RoleMap from nisse_core.state.store import Transcript EXPERT_ROLE = "expert" @@ -168,6 +168,9 @@ class ExpertDesk: model=model, system=system, ) + except ModelRefused as exc: + # Not retried: the same question is declined again. + return None, f"{exc}. Ask it differently, or work from the repository" except LLMError as exc: return None, str(exc) text = "\n".join( diff --git a/nisse-core/src/nisse_core/models/llm.py b/nisse-core/src/nisse_core/models/llm.py index d71b5ff..7f6a815 100644 --- a/nisse-core/src/nisse_core/models/llm.py +++ b/nisse-core/src/nisse_core/models/llm.py @@ -102,6 +102,24 @@ class TruncatedResponse(LLMError): """ +class ModelRefused(LLMError): + """The provider's safety filter declined to answer. + + It comes back as a successful reply with no text (``finish_reason: + content_filter`` on the chat route, ``stop_reason: refusal`` on the + passthrough), so unchecked it reads as an empty answer. It can be a false + positive: Opus declined, every time, a question about which repositories + a Gitea token can list. Asking the same thing again gets the same answer. + """ + + +def _refused(model: str) -> ModelRefused: + return ModelRefused( + f"{model} declined to answer: its provider's safety filter stopped the " + "reply. That says nothing about the subject itself" + ) + + class LLMClient(Protocol): def complete( self, @@ -216,6 +234,8 @@ class LiteLLMClient: f"{requested_model} hit its output limit mid-answer; raise " "llm.max_output_tokens, or the model's max_output_tokens in LiteLLM" ) + if choice.get("finish_reason") == "content_filter": + raise _refused(requested_model) produced_by = data.get("model") or requested_model # What actually answered, which is not always what was asked for: a # proxy fallback answers under another model's name. @@ -338,6 +358,8 @@ class AnthropicPassthroughClient: f"{requested_model} hit its output limit mid-answer; raise " "llm.max_output_tokens, or the model's max_output_tokens in LiteLLM" ) + if data.get("stop_reason") == "refusal": + raise _refused(requested_model) content = data.get("content") if not isinstance(content, list): raise LLMError(f"malformed passthrough response: {str(data)[:300]}") diff --git a/nisse-core/tests/test_expert.py b/nisse-core/tests/test_expert.py index 014cd39..ca46309 100644 --- a/nisse-core/tests/test_expert.py +++ b/nisse-core/tests/test_expert.py @@ -18,7 +18,7 @@ from nisse_core.loop.expert import ( ExpertDesk, parse_answer, ) -from nisse_core.models.llm import LLMError, RoleMap +from nisse_core.models.llm import LLMError, RoleMap, _refused from nisse_core.oplog import OpLog from nisse_core.state.store import Transcript from nisse_dev.state import StateStore @@ -142,6 +142,33 @@ def test_unparseable_reply_is_retried_once_then_raises(store: StateStore) -> Non assert "not a single valid JSON object" in client.calls[1]["system"] +class DecliningClient(ScriptedClient): + """The provider's safety filter, declining every time.""" + + def complete(self, events: Any, tools: Any, model: str, system: Any = None) -> Any: + self.calls.append({"model": model, "system": system}) + raise _refused(model) # the clients' own wording + + +def test_a_declined_question_is_not_asked_again_and_says_why( + store: StateStore, +) -> None: + client = DecliningClient([]) + with pytest.raises(LLMError) as raised: + desk(client, store).ask("which repositories can a token list?") + assert len(client.calls) == 1 # the same question is declined again + assert "Ask it differently, or work from the repository" in str(raised.value) + [completed] = [ + e.event for e in log(store).read() if isinstance(e.event, DelegationCompleted) + ] + assert completed.exit_status == "failed" + assert completed.handback == ( + "claude-haiku-4-5 declined to answer: its provider's safety filter stopped the " + "reply. That says nothing about the subject itself. Ask it " + "differently, or work from the repository" + ) + + def test_retry_succeeds_after_a_bad_first_reply(store: StateStore) -> None: client = ScriptedClient(["waffle", GOOD]) assert desk(client, store).ask("q").confidence == "high" diff --git a/nisse-core/tests/test_llm.py b/nisse-core/tests/test_llm.py index 46aad3c..71458a6 100644 --- a/nisse-core/tests/test_llm.py +++ b/nisse-core/tests/test_llm.py @@ -20,6 +20,7 @@ from nisse_core.models.llm import ( BudgetExhausted, LiteLLMClient, LLMError, + ModelRefused, RoleMap, TruncatedResponse, UpstreamUnavailable, @@ -295,6 +296,51 @@ def test_anthropic_route_raises_on_truncation() -> None: client.complete(EVENTS, tools=None, model="claude-opus-5") +# --- the provider's safety filter ------------------------------------------- + + +def test_openai_route_raises_when_the_filter_declines() -> None: + # As LiteLLM reports Opus's refusal (seen live, #164 follow-up): a 200 + # with no text, which used to read as an empty answer. + declined = httpx.Response( + 200, + json={ + "model": "claude-opus-5-5", + "choices": [ + { + "finish_reason": "content_filter", + "message": {"role": "assistant", "content": None}, + } + ], + "usage": {"prompt_tokens": 612, "completion_tokens": 0}, + }, + ) + client = make_client(lambda request: declined) + with pytest.raises(ModelRefused, match="safety filter"): + client.complete(EVENTS, tools=None, model="claude-opus-5-5") + + +def test_anthropic_route_raises_when_the_filter_declines() -> None: + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"model": "claude-opus-5-5", "stop_reason": "refusal", "content": []}, + ) + + client = AnthropicPassthroughClient( + "https://litellm.test", + "tok", + transport=httpx.MockTransport(handler), + sleep=lambda _s: None, + ) + with pytest.raises(ModelRefused, match="declined to answer"): + client.complete(EVENTS, tools=None, model="claude-opus-5-5") + + +def test_a_refusal_is_an_llm_error_so_callers_already_handle_it() -> None: + assert issubclass(ModelRefused, LLMError) + + def test_a_complete_answer_is_not_mistaken_for_truncation() -> None: def handler(request: httpx.Request) -> httpx.Response: return httpx.Response(