Enforce the tool-call contract, asking servers only where it works

The executor ended a milestone on any reply without a tool call, and
nothing held a model to the offered tools. Measured on the 4070:
tool_choice "required" is enforced by vLLM (local-fast) and Anthropic,
but qwen3-14b (local-coder) under it cannot stop and writes until the
output ceiling, a known model bug. So enforcement is asked for per
model, and the loop holds the contract itself everywhere.

- The catalog reads supports_tool_choice from LiteLLM's model_info;
  BudgetedClient sends tool_choice only to models declared true, as
  "required" on the normalized route and {"type": "any"} on the
  Anthropic passthrough, and only when tools are offered
- The executor gains an explicit finish tool, the only way to end a
  milestone. Finishing with nothing changed is refused once; a prose
  reply gets one nudge, then escalates; a call to a tool not offered
  counts toward failure_limit, like a failed command; calls after
  finish are recorded as not run
- Clients record the model that actually answered. ModelFacts.answered
  recognises a proxy fallback, which once credited Claude's answers to
  a local model that could not call tools at all
- The context probe reports a fallback instead of trusting it; rerun,
  both local models now refuse over-long prompts, as they should
- The capability matrix checks every answer came from the model asked,
  that models call only offered tools, and that a declared
  supports_tool_choice is true
- Spec A15 and the tuning guide say how enforcement is requested,
  declared and backed up

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

nisse-dev/tests/test_executor.py

@@ -4,7 +4,13 @@ from typing import Any
 
 import pytest
 
-from nisse_dev.canonical import AssistantText, CanonicalEvent, StoredEvent, ToolCall
+from nisse_dev.canonical import (
+    AssistantText,
+    CanonicalEvent,
+    StoredEvent,
+    ToolCall,
+    ToolResult,
+)
 from nisse_dev.executor import LLMExecutor
 from nisse_dev.llm import LLMError, RoleMap
 from nisse_dev.oplog import OpLog
@@ -33,7 +39,7 @@ class ScriptedClient:
     ) -> list[CanonicalEvent]:
         self.calls.append({"tools": tools, "model": model, "system": system})
         if not self._turns:
-            return [AssistantText(text="done", model=model)]
+            return [finish(call_id=f"fin-{len(self.calls)}")]
         return self._turns.pop(0)
 
 
@@ -46,6 +52,10 @@ def call(tool: str, call_id: str = "c1", **arguments: Any) -> ToolCall:
     return ToolCall(call_id=call_id, tool=tool, arguments=arguments)
 
 
+def finish(summary: str = "done", call_id: str = "fin") -> ToolCall:
+    return ToolCall(call_id=call_id, tool="finish", arguments={"summary": summary})
+
+
 @pytest.fixture
 def store(tmp_path: Path) -> StateStore:
     return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
@@ -66,15 +76,16 @@ def test_tool_calls_are_executed_and_results_logged(
     client = ScriptedClient(
         [
             [call("write_file", path="app.py", content="print(1)\n")],
-            [AssistantText(text="wrote the file", model="llama3.2")],
+            [finish("wrote the file")],
         ]
     )
     outcome = build(client, store, ws).run(milestone=1)
 
     assert outcome.status == "completed"
+    assert outcome.note == "wrote the file"
     assert (ws.root / "app.py").read_text() == "print(1)\n"
     kinds = [type(e.event).__name__ for e in store.read_events()]
-    assert kinds == ["ToolCall", "ToolResult", "AssistantText"]
+    assert kinds == ["ToolCall", "ToolResult", "ToolCall", "ToolResult"]
 
 
 def test_uses_executor_role_and_reduced_manifest(
@@ -84,50 +95,99 @@ def test_uses_executor_role_and_reduced_manifest(
     build(client, store, ws).run(milestone=2)
     assert client.calls[0]["model"] == "llama3.2"  # never the planner
     names = [t["function"]["name"] for t in client.calls[0]["tools"]]
-    assert names == ["read_file", "write_file", "list_dir", "run_command"]
+    assert names == ["read_file", "write_file", "list_dir", "run_command", "finish"]
     assert "milestone 2" in client.calls[0]["system"]
 
 
-def test_no_tool_call_after_work_ends_the_milestone(
-    store: StateStore, ws: Workspace
-) -> None:
+def test_prose_no_longer_ends_the_milestone(store: StateStore, ws: Workspace) -> None:
+    # Only finish ends a milestone. Prose breaks the tool contract: the model
+    # is told once, and here it gets back to work.
     client = ScriptedClient(
         [
             [call("write_file", path="a.py", content="x")],
             [AssistantText(text="all steps done", model="m")],
+            [finish()],
         ]
     )
     assert build(client, store, ws).run(milestone=1).status == "completed"
+    assert "Every reply must be a tool call" in client.calls[2]["system"]
 
 
-def test_completion_claimed_without_any_work_is_challenged(
+def test_persisting_in_prose_escalates(store: StateStore, ws: Workspace) -> None:
+    prose = [AssistantText(text="let me just chat", model="m")]
+    client = ScriptedClient([prose, prose])
+    outcome = build(client, store, ws).run(milestone=1)
+    assert outcome.status == "escalate"
+    assert "prose" in outcome.note
+
+
+def test_finishing_without_any_work_is_refused_once(
     store: StateStore, ws: Workspace
 ) -> None:
-    # A model that answers with prose on turn one has done nothing; believing
-    # it would record an empty milestone as a success.
+    # A model that finishes on turn one has done nothing; believing it would
+    # record an empty milestone as a success.
     client = ScriptedClient(
         [
-            [AssistantText(text="looks done to me", model="m")],
+            [finish("looks done to me", call_id="f1")],
             [call("write_file", path="a.py", content="x")],
-            [AssistantText(text="now it is done", model="m")],
+            [finish("now it is done", call_id="f2")],
         ]
     )
     outcome = build(client, store, ws).run(milestone=1)
     assert outcome.status == "completed"
-    assert (ws.root / "a.py").exists()  # the nudge produced actual work
-    assert "not made any change yet" in client.calls[1]["system"]
+    assert (ws.root / "a.py").exists()  # the refusal produced actual work
+    [refusal] = [
+        e.event
+        for e in store.read_events()
+        if isinstance(e.event, ToolResult) and e.event.call_id == "f1"
+    ]
+    assert not refusal.success
+    assert "not made any change yet" in refusal.result
 
 
-def test_persisting_in_an_empty_claim_escalates(
+def test_persisting_in_an_empty_finish_escalates(
     store: StateStore, ws: Workspace
 ) -> None:
-    prose = [AssistantText(text="nothing to do", model="m")]
-    client = ScriptedClient([prose, prose])
+    client = ScriptedClient([[finish(call_id="f1")], [finish(call_id="f2")]])
     outcome = build(client, store, ws).run(milestone=1)
     assert outcome.status == "escalate"
     assert "without making any change" in outcome.note
 
 
+def test_calls_after_finish_are_not_run(store: StateStore, ws: Workspace) -> None:
+    client = ScriptedClient(
+        [
+            [call("write_file", path="a.py", content="x")],
+            [finish(), call("write_file", call_id="late", path="b.py", content="y")],
+        ]
+    )
+    assert build(client, store, ws).run(milestone=1).status == "completed"
+    assert not (ws.root / "b.py").exists()
+    [late] = [
+        e.event
+        for e in store.read_events()
+        if isinstance(e.event, ToolResult) and e.event.call_id == "late"
+    ]
+    assert "finish had already ended the milestone" in late.result
+
+
+def test_an_invented_tool_counts_toward_escalation(
+    store: StateStore, ws: Workspace
+) -> None:
+    # Measured: a model on a server that doesn't enforce tool_choice will
+    # call tools that were never offered ("chat", "respond"). Without this it
+    # would burn every turn up to the cap.
+    client = ScriptedClient(
+        [[call("chat", call_id="x1")], [call("respond_user", call_id="x2")]]
+    )
+    outcome = build(client, store, ws, failure_limit=2).run(milestone=1)
+    assert outcome.status == "escalate"
+    assert len(client.calls) == 2
+    results = [e.event for e in store.read_events() if isinstance(e.event, ToolResult)]
+    assert "no such tool: chat" in results[0].result
+    assert "finish" in results[0].result  # it is told what does exist
+
+
 def test_two_consecutive_command_failures_escalate(
     store: StateStore, ws: Workspace
 ) -> None:
@@ -150,7 +210,7 @@ def test_a_success_resets_the_failure_count(store: StateStore, ws: Workspace) ->
             [call("run_command", command="exit 1")],
             [call("run_command", command="true")],
             [call("run_command", command="exit 1")],
-            [AssistantText(text="done", model="m")],
+            [finish()],
         ]
     )
     assert build(client, store, ws, failure_limit=2).run(milestone=1).status == (
@@ -168,7 +228,7 @@ def test_failed_reads_do_not_count_toward_escalation(
             [call("read_file", path="missing-a.txt")],
             [call("read_file", path="missing-b.txt")],
             [call("write_file", path="a.py", content="x")],
-            [AssistantText(text="done", model="m")],
+            [finish()],
         ]
     )
     assert build(client, store, ws, failure_limit=2).run(milestone=1).status == (
@@ -182,8 +242,8 @@ def test_reading_alone_is_not_doing_the_work(store: StateStore, ws: Workspace) -
     client = ScriptedClient(
         [
             [call("list_dir", path=".")],
-            [AssistantText(text="looks done to me", model="m")],
-            [AssistantText(text="really, done", model="m")],
+            [finish("looks done to me", call_id="f1")],
+            [finish("really, done", call_id="f2")],
         ]
     )
     outcome = build(client, store, ws).run(milestone=1)
@@ -214,7 +274,7 @@ def test_multiple_calls_in_one_turn_all_run(store: StateStore, ws: Workspace) ->
                 call("write_file", call_id="a", path="one.txt", content="1"),
                 call("write_file", call_id="b", path="two.txt", content="2"),
             ],
-            [AssistantText(text="done", model="m")],
+            [finish()],
         ]
     )
     build(client, store, ws).run(milestone=1)
@@ -222,7 +282,7 @@ def test_multiple_calls_in_one_turn_all_run(store: StateStore, ws: Workspace) ->
     results = [
         e.event for e in store.read_events() if type(e.event).__name__ == "ToolResult"
     ]
-    assert {r.call_id for r in results} == {"a", "b"}
+    assert {r.call_id for r in results} == {"a", "b", "fin"}
 
 
 class StubDesk:
@@ -255,7 +315,8 @@ def test_ask_expert_is_offered_only_when_a_desk_exists(
     client2 = ScriptedClient([])
     build(client2, store, ws, expert=StubDesk()).run(milestone=1)
     names = [t["function"]["name"] for t in client2.calls[0]["tools"]]
-    assert names[-1] == "ask_expert"
+    assert "ask_expert" in names
+    assert names[-1] == "finish"
     # A12: in addition to the observable triggers, not instead of them.
     assert "run_command" in names
 
@@ -268,7 +329,7 @@ def test_ask_expert_answer_comes_back_as_a_tool_result(
         [
             [call("ask_expert", question="sqlite or postgres?")],
             [call("write_file", path="a.py", content="x")],
-            [AssistantText(text="done", model="m")],
+            [finish()],
         ]
     )
     build(client, store, ws, expert=desk).run(milestone=1)
@@ -286,8 +347,8 @@ def test_consulting_is_not_doing_the_work(store: StateStore, ws: Workspace) -> N
     client = ScriptedClient(
         [
             [call("ask_expert", question="what should I do?")],
-            [AssistantText(text="I think it is done", model="m")],
-            [AssistantText(text="really done", model="m")],
+            [finish("I think it is done", call_id="f1")],
+            [finish("really done", call_id="f2")],
         ]
     )
     outcome = build(client, store, ws, expert=StubDesk()).run(milestone=1)
@@ -304,7 +365,7 @@ def test_exhausted_budget_is_reported_to_the_model_not_raised(
         [
             [call("ask_expert", question="again?")],
             [call("write_file", path="a.py", content="x")],
-            [AssistantText(text="done", model="m")],
+            [finish()],
         ]
     )
     outcome = build(client, store, ws, expert=desk).run(milestone=1)