A model's safety-filter refusal is an error that says so, not an empty reply
Opus's provider filter declined one of nisse-review's ask_expert questions (which repositories a Gitea token can list) every time. It came back as a 200 with no text, and the expert reported "unparseable reply: empty reply". Both LLM clients now raise ModelRefused for finish_reason content_filter (chat route) and stop_reason refusal (passthrough). The expert's handback says what happened and what to do instead. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
This commit is contained in:
1 parent
098d42ff2c
commit
9d2992c20a
4 files changed
+96
-2
No files matched your search
@@ -18,7 +18,7 @@ from nisse_core.loop.expert import (
|
||||
ExpertDesk,
|
||||
parse_answer,
|
||||
)
|
||||
from nisse_core.models.llm import LLMError, RoleMap
|
||||
from nisse_core.models.llm import LLMError, ModelRefused, RoleMap
|
||||
from nisse_core.oplog import OpLog
|
||||
from nisse_core.state.store import Transcript
|
||||
from nisse_dev.state import StateStore
|
||||
@@ -142,6 +142,29 @@ def test_unparseable_reply_is_retried_once_then_raises(store: StateStore) -> Non
|
||||
assert "not a single valid JSON object" in client.calls[1]["system"]
|
||||
|
||||
|
||||
class DecliningClient(ScriptedClient):
|
||||
"""The provider's safety filter, declining every time."""
|
||||
|
||||
def complete(self, events: Any, tools: Any, model: str, system: Any = None) -> Any:
|
||||
self.calls.append({"model": model, "system": system})
|
||||
raise ModelRefused(f"{model} declined to answer: its provider's safety filter")
|
||||
|
||||
|
||||
def test_a_declined_question_is_not_asked_again_and_says_why(
|
||||
store: StateStore,
|
||||
) -> None:
|
||||
client = DecliningClient([])
|
||||
with pytest.raises(LLMError) as raised:
|
||||
desk(client, store).ask("which repositories can a token list?")
|
||||
assert len(client.calls) == 1 # the same question is declined again
|
||||
assert "Ask it differently, or work from the repository" in str(raised.value)
|
||||
[completed] = [
|
||||
e.event for e in log(store).read() if isinstance(e.event, DelegationCompleted)
|
||||
]
|
||||
assert completed.exit_status == "failed"
|
||||
assert "safety filter" in completed.handback
|
||||
|
||||
|
||||
def test_retry_succeeds_after_a_bad_first_reply(store: StateStore) -> None:
|
||||
client = ScriptedClient(["waffle", GOOD])
|
||||
assert desk(client, store).ask("q").confidence == "high"
|
||||
|
||||
Reference in new issue
Block a user