A model's safety-filter refusal is an error that says so, not an empty reply
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 2m25s
CI / e2e (pull_request) Successful in 4m41s

Opus's provider filter declined one of nisse-review's ask_expert
questions (which repositories a Gitea token can list) every time. It came
back as a 200 with no text, and the expert reported "unparseable reply:
empty reply". Both LLM clients now raise ModelRefused for
finish_reason content_filter (chat route) and stop_reason refusal
(passthrough). The expert's handback says what happened and what to do
instead.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-10-18 13:39:26 +00:00
1 parent 098d42ff2c
commit 9d2992c20a
4 files changed
+96 -2

No files matched your search

+24 -1
View File
@@ -18,7 +18,7 @@ from nisse_core.loop.expert import (
ExpertDesk,
parse_answer,
)
from nisse_core.models.llm import LLMError, RoleMap
from nisse_core.models.llm import LLMError, ModelRefused, RoleMap
from nisse_core.oplog import OpLog
from nisse_core.state.store import Transcript
from nisse_dev.state import StateStore
@@ -142,6 +142,29 @@ def test_unparseable_reply_is_retried_once_then_raises(store: StateStore) -> Non
assert "not a single valid JSON object" in client.calls[1]["system"]
class DecliningClient(ScriptedClient):
"""The provider's safety filter, declining every time."""
def complete(self, events: Any, tools: Any, model: str, system: Any = None) -> Any:
self.calls.append({"model": model, "system": system})
raise ModelRefused(f"{model} declined to answer: its provider's safety filter")
def test_a_declined_question_is_not_asked_again_and_says_why(
store: StateStore,
) -> None:
client = DecliningClient([])
with pytest.raises(LLMError) as raised:
desk(client, store).ask("which repositories can a token list?")
assert len(client.calls) == 1 # the same question is declined again
assert "Ask it differently, or work from the repository" in str(raised.value)
[completed] = [
e.event for e in log(store).read() if isinstance(e.event, DelegationCompleted)
]
assert completed.exit_status == "failed"
assert "safety filter" in completed.handback
def test_retry_succeeds_after_a_bad_first_reply(store: StateStore) -> None:
client = ScriptedClient(["waffle", GOOD])
assert desk(client, store).ask("q").confidence == "high"