Enforce the tool-call contract, asking servers only where it works

The executor ended a milestone on any reply without a tool call, and
nothing held a model to the offered tools. Measured on the 4070:
tool_choice "required" is enforced by vLLM (local-fast) and Anthropic,
but qwen3-14b (local-coder) under it cannot stop and writes until the
output ceiling, a known model bug. So enforcement is asked for per
model, and the loop holds the contract itself everywhere.

- The catalog reads supports_tool_choice from LiteLLM's model_info;
  BudgetedClient sends tool_choice only to models declared true, as
  "required" on the normalized route and {"type": "any"} on the
  Anthropic passthrough, and only when tools are offered
- The executor gains an explicit finish tool, the only way to end a
  milestone. Finishing with nothing changed is refused once; a prose
  reply gets one nudge, then escalates; a call to a tool not offered
  counts toward failure_limit, like a failed command; calls after
  finish are recorded as not run
- Clients record the model that actually answered. ModelFacts.answered
  recognises a proxy fallback, which once credited Claude's answers to
  a local model that could not call tools at all
- The context probe reports a fallback instead of trusting it; rerun,
  both local models now refuse over-long prompts, as they should
- The capability matrix checks every answer came from the model asked,
  that models call only offered tools, and that a declared
  supports_tool_choice is true
- Spec A15 and the tuning guide say how enforcement is requested,
  declared and backed up

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

nisse-dev/tests/test_agent_build.py

@@ -344,4 +344,6 @@ def test_tuning_reaches_every_component(
     assert executor._client._inner._max_tokens == 8192
     assert delegation._client._inner._max_tokens == 8192
     assert executor._client._max_output_tokens == 8192
+    # tool_choice is asked of exactly the models LiteLLM declares enforce it.
+    assert executor._client._enforces_tool_choice("claude-opus-5") is False
     agent.stop()

nisse-dev/tests/test_context.py

@@ -150,8 +150,8 @@ class RecordingClient:
         self.calls: list[dict[str, Any]] = []
         self.last_usage = {"input": reported_input, "output": 5}
 
-    def complete(self, events, tools, model, system=None, *, max_tokens=None):
-        self.calls.append({"events": events, "max_tokens": max_tokens})
+    def complete(self, events, tools, model, system=None, *, max_tokens=None, **kwargs):
+        self.calls.append({"events": events, "max_tokens": max_tokens, **kwargs})
         return [AssistantText(text="ok", model=model)]
 
 
@@ -292,3 +292,51 @@ def test_a_request_that_cannot_fit_stops_the_milestone(tmp_path: Path) -> None:
     assert outcome.status == "escalate"
     assert "fit" in outcome.note
     assert inner.calls == []
+
+
+# --- tool_choice is only asked of servers that enforce it -------------------
+
+FINISH = [{"type": "function", "function": {"name": "finish", "parameters": {}}}]
+
+
+def test_required_is_asked_only_of_models_declared_to_enforce_it() -> None:
+    # Measured: qwen3-14b under "required" cannot stop and rambles to the
+    # output ceiling, so a model that is not declared is never asked.
+    inner = RecordingClient()
+    client = BudgetedClient(
+        inner, lambda m: (32768, 8192), enforces_tool_choice=lambda m: m == "local-fast"
+    )
+    client.complete(transcript(1, 10), FINISH, "local-fast")
+    client.complete(transcript(1, 10), FINISH, "local-coder")
+    client.complete(transcript(1, 10), None, "local-fast")
+    assert [c["require_tool_call"] for c in inner.calls] == [True, False, False]
+
+
+def test_the_catalog_reads_supports_tool_choice() -> None:
+    catalog = ModelCatalog.from_payload(
+        {
+            "data": [
+                {"model_name": "a", "model_info": {"supports_tool_choice": True}},
+                {"model_name": "b", "model_info": {"supports_tool_choice": False}},
+                {"model_name": "c", "model_info": {}},
+            ]
+        }
+    )
+    assert [catalog.facts(m).supports_tool_choice for m in "abc"] == [True, False, None]
+
+
+def test_a_fallback_answer_is_recognised() -> None:
+    facts = ModelCatalog.from_payload(
+        {
+            "data": [
+                {
+                    "model_name": "claude-haiku-4-5",
+                    "litellm_params": {"model": "anthropic/claude-haiku-4-5"},
+                }
+            ]
+        }
+    ).facts("claude-haiku-4-5")
+    assert facts.answered("claude-haiku-4-5")  # LiteLLM's alias
+    assert facts.answered("claude-haiku-4-5-20251001")  # the provider's own name
+    assert facts.answered(None)  # nothing reported
+    assert not facts.answered("claude-sonnet-5")  # somebody else answered

nisse-dev/tests/test_executor.py

@@ -4,7 +4,13 @@ from typing import Any
 
 import pytest
 
-from nisse_dev.canonical import AssistantText, CanonicalEvent, StoredEvent, ToolCall
+from nisse_dev.canonical import (
+    AssistantText,
+    CanonicalEvent,
+    StoredEvent,
+    ToolCall,
+    ToolResult,
+)
 from nisse_dev.executor import LLMExecutor
 from nisse_dev.llm import LLMError, RoleMap
 from nisse_dev.oplog import OpLog
@@ -33,7 +39,7 @@ class ScriptedClient:
     ) -> list[CanonicalEvent]:
         self.calls.append({"tools": tools, "model": model, "system": system})
         if not self._turns:
-            return [AssistantText(text="done", model=model)]
+            return [finish(call_id=f"fin-{len(self.calls)}")]
         return self._turns.pop(0)
 
 
@@ -46,6 +52,10 @@ def call(tool: str, call_id: str = "c1", **arguments: Any) -> ToolCall:
     return ToolCall(call_id=call_id, tool=tool, arguments=arguments)
 
 
+def finish(summary: str = "done", call_id: str = "fin") -> ToolCall:
+    return ToolCall(call_id=call_id, tool="finish", arguments={"summary": summary})
+
+
 @pytest.fixture
 def store(tmp_path: Path) -> StateStore:
     return StateStore(tmp_path / "state.db", oplog=OpLog("nisse-dev", io.StringIO()))
@@ -66,15 +76,16 @@ def test_tool_calls_are_executed_and_results_logged(
     client = ScriptedClient(
         [
             [call("write_file", path="app.py", content="print(1)\n")],
-            [AssistantText(text="wrote the file", model="llama3.2")],
+            [finish("wrote the file")],
         ]
     )
     outcome = build(client, store, ws).run(milestone=1)
 
     assert outcome.status == "completed"
+    assert outcome.note == "wrote the file"
     assert (ws.root / "app.py").read_text() == "print(1)\n"
     kinds = [type(e.event).__name__ for e in store.read_events()]
-    assert kinds == ["ToolCall", "ToolResult", "AssistantText"]
+    assert kinds == ["ToolCall", "ToolResult", "ToolCall", "ToolResult"]
 
 
 def test_uses_executor_role_and_reduced_manifest(
@@ -84,50 +95,99 @@ def test_uses_executor_role_and_reduced_manifest(
     build(client, store, ws).run(milestone=2)
     assert client.calls[0]["model"] == "llama3.2"  # never the planner
     names = [t["function"]["name"] for t in client.calls[0]["tools"]]
-    assert names == ["read_file", "write_file", "list_dir", "run_command"]
+    assert names == ["read_file", "write_file", "list_dir", "run_command", "finish"]
     assert "milestone 2" in client.calls[0]["system"]
 
 
-def test_no_tool_call_after_work_ends_the_milestone(
-    store: StateStore, ws: Workspace
-) -> None:
+def test_prose_no_longer_ends_the_milestone(store: StateStore, ws: Workspace) -> None:
+    # Only finish ends a milestone. Prose breaks the tool contract: the model
+    # is told once, and here it gets back to work.
     client = ScriptedClient(
         [
             [call("write_file", path="a.py", content="x")],
             [AssistantText(text="all steps done", model="m")],
+            [finish()],
         ]
     )
     assert build(client, store, ws).run(milestone=1).status == "completed"
+    assert "Every reply must be a tool call" in client.calls[2]["system"]
 
 
-def test_completion_claimed_without_any_work_is_challenged(
+def test_persisting_in_prose_escalates(store: StateStore, ws: Workspace) -> None:
+    prose = [AssistantText(text="let me just chat", model="m")]
+    client = ScriptedClient([prose, prose])
+    outcome = build(client, store, ws).run(milestone=1)
+    assert outcome.status == "escalate"
+    assert "prose" in outcome.note
+
+
+def test_finishing_without_any_work_is_refused_once(
     store: StateStore, ws: Workspace
 ) -> None:
-    # A model that answers with prose on turn one has done nothing; believing
-    # it would record an empty milestone as a success.
+    # A model that finishes on turn one has done nothing; believing it would
+    # record an empty milestone as a success.
     client = ScriptedClient(
         [
-            [AssistantText(text="looks done to me", model="m")],
+            [finish("looks done to me", call_id="f1")],
             [call("write_file", path="a.py", content="x")],
-            [AssistantText(text="now it is done", model="m")],
+            [finish("now it is done", call_id="f2")],
         ]
     )
     outcome = build(client, store, ws).run(milestone=1)
     assert outcome.status == "completed"
-    assert (ws.root / "a.py").exists()  # the nudge produced actual work
-    assert "not made any change yet" in client.calls[1]["system"]
+    assert (ws.root / "a.py").exists()  # the refusal produced actual work
+    [refusal] = [
+        e.event
+        for e in store.read_events()
+        if isinstance(e.event, ToolResult) and e.event.call_id == "f1"
+    ]
+    assert not refusal.success
+    assert "not made any change yet" in refusal.result
 
 
-def test_persisting_in_an_empty_claim_escalates(
+def test_persisting_in_an_empty_finish_escalates(
     store: StateStore, ws: Workspace
 ) -> None:
-    prose = [AssistantText(text="nothing to do", model="m")]
-    client = ScriptedClient([prose, prose])
+    client = ScriptedClient([[finish(call_id="f1")], [finish(call_id="f2")]])
     outcome = build(client, store, ws).run(milestone=1)
     assert outcome.status == "escalate"
     assert "without making any change" in outcome.note
 
 
+def test_calls_after_finish_are_not_run(store: StateStore, ws: Workspace) -> None:
+    client = ScriptedClient(
+        [
+            [call("write_file", path="a.py", content="x")],
+            [finish(), call("write_file", call_id="late", path="b.py", content="y")],
+        ]
+    )
+    assert build(client, store, ws).run(milestone=1).status == "completed"
+    assert not (ws.root / "b.py").exists()
+    [late] = [
+        e.event
+        for e in store.read_events()
+        if isinstance(e.event, ToolResult) and e.event.call_id == "late"
+    ]
+    assert "finish had already ended the milestone" in late.result
+
+
+def test_an_invented_tool_counts_toward_escalation(
+    store: StateStore, ws: Workspace
+) -> None:
+    # Measured: a model on a server that doesn't enforce tool_choice will
+    # call tools that were never offered ("chat", "respond"). Without this it
+    # would burn every turn up to the cap.
+    client = ScriptedClient(
+        [[call("chat", call_id="x1")], [call("respond_user", call_id="x2")]]
+    )
+    outcome = build(client, store, ws, failure_limit=2).run(milestone=1)
+    assert outcome.status == "escalate"
+    assert len(client.calls) == 2
+    results = [e.event for e in store.read_events() if isinstance(e.event, ToolResult)]
+    assert "no such tool: chat" in results[0].result
+    assert "finish" in results[0].result  # it is told what does exist
+
+
 def test_two_consecutive_command_failures_escalate(
     store: StateStore, ws: Workspace
 ) -> None:
@@ -150,7 +210,7 @@ def test_a_success_resets_the_failure_count(store: StateStore, ws: Workspace) ->
             [call("run_command", command="exit 1")],
             [call("run_command", command="true")],
             [call("run_command", command="exit 1")],
-            [AssistantText(text="done", model="m")],
+            [finish()],
         ]
     )
     assert build(client, store, ws, failure_limit=2).run(milestone=1).status == (
@@ -168,7 +228,7 @@ def test_failed_reads_do_not_count_toward_escalation(
             [call("read_file", path="missing-a.txt")],
             [call("read_file", path="missing-b.txt")],
             [call("write_file", path="a.py", content="x")],
-            [AssistantText(text="done", model="m")],
+            [finish()],
         ]
     )
     assert build(client, store, ws, failure_limit=2).run(milestone=1).status == (
@@ -182,8 +242,8 @@ def test_reading_alone_is_not_doing_the_work(store: StateStore, ws: Workspace) -
     client = ScriptedClient(
         [
             [call("list_dir", path=".")],
-            [AssistantText(text="looks done to me", model="m")],
-            [AssistantText(text="really, done", model="m")],
+            [finish("looks done to me", call_id="f1")],
+            [finish("really, done", call_id="f2")],
         ]
     )
     outcome = build(client, store, ws).run(milestone=1)
@@ -214,7 +274,7 @@ def test_multiple_calls_in_one_turn_all_run(store: StateStore, ws: Workspace) ->
                 call("write_file", call_id="a", path="one.txt", content="1"),
                 call("write_file", call_id="b", path="two.txt", content="2"),
             ],
-            [AssistantText(text="done", model="m")],
+            [finish()],
         ]
     )
     build(client, store, ws).run(milestone=1)
@@ -222,7 +282,7 @@ def test_multiple_calls_in_one_turn_all_run(store: StateStore, ws: Workspace) ->
     results = [
         e.event for e in store.read_events() if type(e.event).__name__ == "ToolResult"
     ]
-    assert {r.call_id for r in results} == {"a", "b"}
+    assert {r.call_id for r in results} == {"a", "b", "fin"}
 
 
 class StubDesk:
@@ -255,7 +315,8 @@ def test_ask_expert_is_offered_only_when_a_desk_exists(
     client2 = ScriptedClient([])
     build(client2, store, ws, expert=StubDesk()).run(milestone=1)
     names = [t["function"]["name"] for t in client2.calls[0]["tools"]]
-    assert names[-1] == "ask_expert"
+    assert "ask_expert" in names
+    assert names[-1] == "finish"
     # A12: in addition to the observable triggers, not instead of them.
     assert "run_command" in names
 
@@ -268,7 +329,7 @@ def test_ask_expert_answer_comes_back_as_a_tool_result(
         [
             [call("ask_expert", question="sqlite or postgres?")],
             [call("write_file", path="a.py", content="x")],
-            [AssistantText(text="done", model="m")],
+            [finish()],
         ]
     )
     build(client, store, ws, expert=desk).run(milestone=1)
@@ -286,8 +347,8 @@ def test_consulting_is_not_doing_the_work(store: StateStore, ws: Workspace) -> N
     client = ScriptedClient(
         [
             [call("ask_expert", question="what should I do?")],
-            [AssistantText(text="I think it is done", model="m")],
-            [AssistantText(text="really done", model="m")],
+            [finish("I think it is done", call_id="f1")],
+            [finish("really done", call_id="f2")],
         ]
     )
     outcome = build(client, store, ws, expert=StubDesk()).run(milestone=1)
@@ -304,7 +365,7 @@ def test_exhausted_budget_is_reported_to_the_model_not_raised(
         [
             [call("ask_expert", question="again?")],
             [call("write_file", path="a.py", content="x")],
-            [AssistantText(text="done", model="m")],
+            [finish()],
         ]
     )
     outcome = build(client, store, ws, expert=desk).run(milestone=1)

nisse-dev/tests/test_llm.py

@@ -1,5 +1,6 @@
 import json
 from pathlib import Path
+from typing import Any
 
 import httpx
 import pytest
@@ -341,3 +342,73 @@ def test_budget_refusal_is_a_kind_of_upstream_trouble() -> None:
     # So one catch covers the family, but it can still be told apart where
     # the difference matters — it will not clear within the hour.
     assert issubclass(BudgetExhausted, UpstreamUnavailable)
+
+
+# --- tool_choice (A15: constrained decoding) --------------------------------
+
+TOOLS = [
+    {
+        "type": "function",
+        "function": {
+            "name": "finish",
+            "description": "done",
+            "parameters": {"type": "object", "properties": {}},
+        },
+    }
+]
+
+
+def capture_bodies() -> tuple[list[dict], Any]:
+    bodies: list[dict] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        bodies.append(json.loads(request.content))
+        return completion({"role": "assistant", "content": "ok"})
+
+    return bodies, handler
+
+
+def test_required_is_sent_only_when_asked_and_tools_are_offered() -> None:
+    bodies, handler = capture_bodies()
+    client = make_client(handler)
+    client.complete(EVENTS, tools=TOOLS, model="m", require_tool_call=True)
+    client.complete(EVENTS, tools=TOOLS, model="m")
+    client.complete(EVENTS, tools=None, model="m", require_tool_call=True)
+    assert bodies[0]["tool_choice"] == "required"
+    assert "tool_choice" not in bodies[1]  # the server's default, auto
+    assert "tool_choice" not in bodies[2]  # nothing to choose from
+
+
+def test_the_passthrough_asks_in_anthropics_terms() -> None:
+    bodies: list[dict] = []
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        bodies.append(json.loads(request.content))
+        return httpx.Response(
+            200,
+            json={
+                "model": "claude-haiku-4-5",
+                "stop_reason": "tool_use",
+                "content": [
+                    {"type": "tool_use", "id": "t", "name": "finish", "input": {}}
+                ],
+            },
+        )
+
+    client = AnthropicPassthroughClient(
+        "https://litellm.test", "tok", transport=httpx.MockTransport(handler)
+    )
+    client.complete(
+        EVENTS, tools=TOOLS, model="claude-haiku-4-5", require_tool_call=True
+    )
+    assert bodies[0]["tool_choice"] == {"type": "any"}
+
+
+def test_the_answering_model_is_recorded() -> None:
+    # A proxy fallback answers under another model's name; this is how a
+    # caller can tell.
+    client = make_client(
+        lambda request: completion({"role": "assistant", "content": "hi"})
+    )
+    client.complete(EVENTS, tools=None, model="local-fast")
+    assert client.last_model == "gpt-test-1"

nisse-dev/tests/test_probe.py

@@ -42,7 +42,9 @@ def test_probe_calibrates_then_sends_half_again_the_window() -> None:
         words = len(json.loads(request.content)["messages"][0]["content"].split())
         sizes.append(words * 2)
         reported = min(words * 2, 4096)
-        return httpx.Response(200, json={"usage": {"prompt_tokens": reported}})
+        return httpx.Response(
+            200, json={"model": "llama3.2", "usage": {"prompt_tokens": reported}}
+        )
 
     with httpx.Client(
         base_url="http://litellm", transport=httpx.MockTransport(handler)
@@ -51,3 +53,37 @@ def test_probe_calibrates_then_sends_half_again_the_window() -> None:
 
     assert result.verdict == "matches"
     assert sizes[1] >= 4096 * 1.4  # the probe really exceeded the window
+
+
+def test_a_fallback_answer_is_never_credited_to_the_model() -> None:
+    """Measured: LiteLLM answered an oversized prompt to local-coder with
+    claude-sonnet-5, and the probe called the refusing server roomy."""
+
+    def handler(request: httpx.Request) -> httpx.Response:
+        words = len(json.loads(request.content)["messages"][0]["content"].split())
+        model = "local-coder" if words < 1000 else "claude-sonnet-5"
+        return httpx.Response(
+            200, json={"model": model, "usage": {"prompt_tokens": words * 2}}
+        )
+
+    with httpx.Client(
+        base_url="http://litellm", transport=httpx.MockTransport(handler)
+    ) as http:
+        result = probe(http, "local-coder", declared=32768, backing="openai/qwen3-14b")
+
+    assert result.verdict == "fallback"
+    assert "claude-sonnet-5 answered instead of local-coder" in result.message
+
+
+def test_a_provider_reporting_its_own_model_name_is_not_a_fallback() -> None:
+    def handler(request: httpx.Request) -> httpx.Response:
+        return httpx.Response(
+            200, json={"model": "qwen3-14b", "usage": {"prompt_tokens": 400}}
+        )
+
+    with httpx.Client(
+        base_url="http://litellm", transport=httpx.MockTransport(handler)
+    ) as http:
+        result = probe(http, "local-coder", declared=32768, backing="openai/qwen3-14b")
+
+    assert result.verdict != "fallback"