Merge pull request 'A model's safety-filter refusal is an error that says so, not an empty reply' (#165) from expert-refusal into main

nisse-core/src/nisse_core/loop/expert.py

@@ -32,7 +32,7 @@ from nisse_core.log.canonical import (
     StoredEvent,
     UserMessage,
 )
-from nisse_core.models.llm import LLMClient, LLMError, RoleMap
+from nisse_core.models.llm import LLMClient, LLMError, ModelRefused, RoleMap
 from nisse_core.state.store import Transcript
 
 EXPERT_ROLE = "expert"
@@ -168,6 +168,9 @@ class ExpertDesk:
                     model=model,
                     system=system,
                 )
+            except ModelRefused as exc:
+                # Not retried: the same question is declined again.
+                return None, f"{exc}. Ask it differently, or work from the repository"
             except LLMError as exc:
                 return None, str(exc)
             text = "\n".join(

nisse-core/src/nisse_core/models/llm.py

@@ -102,6 +102,24 @@ class TruncatedResponse(LLMError):
     """
 
 
+class ModelRefused(LLMError):
+    """The provider's safety filter declined to answer.
+
+    It comes back as a successful reply with no text (``finish_reason:
+    content_filter`` on the chat route, ``stop_reason: refusal`` on the
+    passthrough), so unchecked it reads as an empty answer. It can be a false
+    positive: Opus declined, every time, a question about which repositories
+    a Gitea token can list. Asking the same thing again gets the same answer.
+    """
+
+
+def _refused(model: str) -> ModelRefused:
+    return ModelRefused(
+        f"{model} declined to answer: its provider's safety filter stopped the "
+        "reply. That says nothing about the subject itself"
+    )
+
+
 class LLMClient(Protocol):
     def complete(
         self,
@@ -216,6 +234,8 @@ class LiteLLMClient:
                 f"{requested_model} hit its output limit mid-answer; raise "
                 "llm.max_output_tokens, or the model's max_output_tokens in LiteLLM"
             )
+        if choice.get("finish_reason") == "content_filter":
+            raise _refused(requested_model)
         produced_by = data.get("model") or requested_model
         # What actually answered, which is not always what was asked for: a
         # proxy fallback answers under another model's name.
@@ -338,6 +358,8 @@ class AnthropicPassthroughClient:
                 f"{requested_model} hit its output limit mid-answer; raise "
                 "llm.max_output_tokens, or the model's max_output_tokens in LiteLLM"
             )
+        if data.get("stop_reason") == "refusal":
+            raise _refused(requested_model)
         content = data.get("content")
         if not isinstance(content, list):
             raise LLMError(f"malformed passthrough response: {str(data)[:300]}")

nisse-core/tests/test_expert.py

@@ -18,7 +18,7 @@ from nisse_core.loop.expert import (
     ExpertDesk,
     parse_answer,
 )
-from nisse_core.models.llm import LLMError, RoleMap
+from nisse_core.models.llm import LLMError, RoleMap, _refused
 from nisse_core.oplog import OpLog
 from nisse_core.state.store import Transcript
 from nisse_dev.state import StateStore
@@ -142,6 +142,33 @@ def test_unparseable_reply_is_retried_once_then_raises(store: StateStore) -> Non
     assert "not a single valid JSON object" in client.calls[1]["system"]
 
 
+class DecliningClient(ScriptedClient):
+    """The provider's safety filter, declining every time."""
+
+    def complete(self, events: Any, tools: Any, model: str, system: Any = None) -> Any:
+        self.calls.append({"model": model, "system": system})
+        raise _refused(model)  # the clients' own wording
+
+
+def test_a_declined_question_is_not_asked_again_and_says_why(
+    store: StateStore,
+) -> None:
+    client = DecliningClient([])
+    with pytest.raises(LLMError) as raised:
+        desk(client, store).ask("which repositories can a token list?")
+    assert len(client.calls) == 1  # the same question is declined again
+    assert "Ask it differently, or work from the repository" in str(raised.value)
+    [completed] = [
+        e.event for e in log(store).read() if isinstance(e.event, DelegationCompleted)
+    ]
+    assert completed.exit_status == "failed"
+    assert completed.handback == (
+        "claude-haiku-4-5 declined to answer: its provider's safety filter stopped the "
+        "reply. That says nothing about the subject itself. Ask it "
+        "differently, or work from the repository"
+    )
+
+
 def test_retry_succeeds_after_a_bad_first_reply(store: StateStore) -> None:
     client = ScriptedClient(["waffle", GOOD])
     assert desk(client, store).ask("q").confidence == "high"

nisse-core/tests/test_llm.py

@@ -20,6 +20,7 @@ from nisse_core.models.llm import (
     BudgetExhausted,
     LiteLLMClient,
     LLMError,
+    ModelRefused,
     RoleMap,
     TruncatedResponse,
     UpstreamUnavailable,
@@ -295,6 +296,51 @@ def test_anthropic_route_raises_on_truncation() -> None:
         client.complete(EVENTS, tools=None, model="claude-opus-5")
 
 
+# --- the provider's safety filter -------------------------------------------
+
+
+def test_openai_route_raises_when_the_filter_declines() -> None:
+    # As LiteLLM reports Opus's refusal (seen live, #164 follow-up): a 200
+    # with no text, which used to read as an empty answer.
+    declined = httpx.Response(
+        200,
+        json={
+            "model": "claude-opus-5-5",
+            "choices": [
+                {
+                    "finish_reason": "content_filter",
+                    "message": {"role": "assistant", "content": None},
+                }
+            ],
+            "usage": {"prompt_tokens": 612, "completion_tokens": 0},
+        },
+    )
+    client = make_client(lambda request: declined)
+    with pytest.raises(ModelRefused, match="safety filter"):
+        client.complete(EVENTS, tools=None, model="claude-opus-5-5")
+
+
+def test_anthropic_route_raises_when_the_filter_declines() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            200,
+            json={"model": "claude-opus-5-5", "stop_reason": "refusal", "content": []},
+        )
+
+    client = AnthropicPassthroughClient(
+        "https://litellm.test",
+        "tok",
+        transport=httpx.MockTransport(handler),
+        sleep=lambda _s: None,
+    )
+    with pytest.raises(ModelRefused, match="declined to answer"):
+        client.complete(EVENTS, tools=None, model="claude-opus-5-5")
+
+
+def test_a_refusal_is_an_llm_error_so_callers_already_handle_it() -> None:
+    assert issubclass(ModelRefused, LLMError)
+
+
 def test_a_complete_answer_is_not_mistaken_for_truncation() -> None:
     def handler(request: httpx.Request) -> httpx.Response:
         return httpx.Response(