Enforce the tool-call contract, asking servers only where it works

The executor ended a milestone on any reply without a tool call, and
nothing held a model to the offered tools. Measured on the 4070:
tool_choice "required" is enforced by vLLM (local-fast) and Anthropic,
but qwen3-14b (local-coder) under it cannot stop and writes until the
output ceiling, a known model bug. So enforcement is asked for per
model, and the loop holds the contract itself everywhere.

- The catalog reads supports_tool_choice from LiteLLM's model_info;
  BudgetedClient sends tool_choice only to models declared true, as
  "required" on the normalized route and {"type": "any"} on the
  Anthropic passthrough, and only when tools are offered
- The executor gains an explicit finish tool, the only way to end a
  milestone. Finishing with nothing changed is refused once; a prose
  reply gets one nudge, then escalates; a call to a tool not offered
  counts toward failure_limit, like a failed command; calls after
  finish are recorded as not run
- Clients record the model that actually answered. ModelFacts.answered
  recognises a proxy fallback, which once credited Claude's answers to
  a local model that could not call tools at all
- The context probe reports a fallback instead of trusting it; rerun,
  both local models now refuse over-long prompts, as they should
- The capability matrix checks every answer came from the model asked,
  that models call only offered tools, and that a declared
  supports_tool_choice is true
- Spec A15 and the tuning guide say how enforcement is requested,
  declared and backed up

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

nisse-dev/src/nisse_dev/agent.py

@@ -174,6 +174,9 @@ def build_agent(
             max_output_tokens=config.max_output_tokens,
             keep_recent_turns=tuning.context.keep_recent_turns,
             safety_margin=tuning.context.safety_margin,
+            enforces_tool_choice=lambda m: (
+                catalog.facts(m).supports_tool_choice is True
+            ),
         )
 
     normalized = budgeted(

nisse-dev/src/nisse_dev/cancel.py

@@ -29,10 +29,12 @@ class Cancel:
         return self._event.is_set()
 
 
-def skip_calls(store: StateStore, calls: list[ToolCall]) -> None:
-    """Record the calls a stop left unrun. Every tool call needs a result for
-    the transcript to render again, and the result says what happened."""
+def skip_calls(
+    store: StateStore, calls: list[ToolCall], reason: str = STOPPED_RESULT
+) -> None:
+    """Record calls left unrun. Every tool call needs a result for the
+    transcript to render again, and the result says what happened."""
     for call in calls:
         store.append_event(
-            ToolResult(call_id=call.call_id, result=STOPPED_RESULT, success=False)
+            ToolResult(call_id=call.call_id, result=reason, success=False)
         )

nisse-dev/src/nisse_dev/catalog.py

@@ -54,11 +54,26 @@ class ModelFacts:
     max_input_tokens: int | None = None
     # The most one response may use (model_info.max_output_tokens).
     max_output_tokens: int | None = None
+    # Whether its server enforces tool_choice (model_info.supports_tool_choice).
+    # Only a declared True counts: a model that says nothing is not asked.
+    supports_tool_choice: bool | None = None
 
     @property
     def native_anthropic(self) -> bool:
         return self.provider == ANTHROPIC_PROVIDER
 
+    def answered(self, reported: str | None) -> bool:
+        """Whether a response reporting this model name came from this model.
+
+        A proxy fallback answers under another model's name. LiteLLM reports
+        the alias; a provider may report its own name for the backing model,
+        sometimes with a date suffix ("claude-haiku-4-5-20251001").
+        """
+        if not reported:
+            return True  # nothing reported, nothing contradicted
+        own_name = self.backing.partition("/")[2] or self.backing
+        return reported in (self.alias, self.backing) or reported.startswith(own_name)
+
 
 class ModelCatalog:
     """What one LiteLLM key can see, as of the moment it was asked."""
@@ -87,6 +102,7 @@ class ModelCatalog:
                 supports_tools=info.get("supports_function_calling"),
                 max_input_tokens=_positive_int(info.get("max_input_tokens")),
                 max_output_tokens=_positive_int(info.get("max_output_tokens")),
+                supports_tool_choice=info.get("supports_tool_choice"),
             )
         return cls(facts)
 

nisse-dev/src/nisse_dev/context.py

@@ -215,9 +215,15 @@ class BudgetedClient:
         max_output_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,
         keep_recent_turns: int = 6,
         safety_margin: float = 0.1,
+        enforces_tool_choice: Callable[[str], bool] = lambda model: False,
     ) -> None:
         self._inner = inner
         self._limits_for = limits_for
+        # Whether a model's server holds it to the offered tools when asked
+        # (LiteLLM's supports_tool_choice). Asking one that doesn't can be
+        # worse than not asking: qwen3-14b can't stop, and rambles until the
+        # output ceiling.
+        self._enforces_tool_choice = enforces_tool_choice
         self._max_output_tokens = max_output_tokens
         self._keep_recent_turns = keep_recent_turns
         self._safety_margin = safety_margin
@@ -229,6 +235,10 @@ class BudgetedClient:
     def last_usage(self) -> dict[str, int]:
         return getattr(self._inner, "last_usage", {}) or {}
 
+    @property
+    def last_model(self) -> str | None:
+        return getattr(self._inner, "last_model", None)
+
     def budget_for(self, model: str) -> Budget:
         window, model_max_output = self._limits_for(model)
         ceiling = self._max_output_tokens
@@ -252,7 +262,15 @@ class BudgetedClient:
         factor = self._factors.get(model, 1.0)
         fitted = fit(events, system, tools, budget, factor)
         produced = self._inner.complete(
-            fitted, tools, model, system=system, max_tokens=budget.max_output
+            fitted,
+            tools,
+            model,
+            system=system,
+            max_tokens=budget.max_output,
+            # Every caller that offers tools needs one called: the executor
+            # finishes with finish, the planner with submit_plan or
+            # ask_human, a delegation with hand_back.
+            require_tool_call=bool(tools) and self._enforces_tool_choice(model),
         )
         reported = int(self.last_usage.get("input", 0))
         if reported > 0:

nisse-dev/src/nisse_dev/executor.py

@@ -27,20 +27,42 @@ EXECUTOR_SYSTEM = """You are nisse-dev's executor, doing mechanical work.
 Follow the approved plan below exactly. It was written to be executed
 verbatim; do not redesign it, expand its scope, or skip steps.
 
-Work by calling tools. Verify your work by running the project's tests.
-When every step of the plan is done and the tests pass, reply with a short
-plain-text summary and no tool call — that ends the milestone."""
+Every reply must be a tool call. Verify your work by running the project's
+tests. When every step of the plan is done and the tests pass, call finish
+with a short summary — that, and nothing else, ends the milestone."""
 
 DEFAULT_MAX_TURNS = 24
 DEFAULT_FAILURE_LIMIT = 2
 
-# A turn with no tool call means "the plan is done". Believing that before
-# any real work has happened would record an empty milestone as a success, so
-# the claim is challenged once first.
-_NUDGE = (
-    "\n\nYou have not made any change yet, so the plan cannot be complete. "
-    "Carry out the next step of the plan with a tool call, or run the tests "
-    "to show the work is already done."
+FINISH_TOOL: dict[str, Any] = {
+    "type": "function",
+    "function": {
+        "name": "finish",
+        "description": (
+            "End the milestone: every step of the plan is done and the tests pass."
+        ),
+        "parameters": {
+            "type": "object",
+            "properties": {
+                "summary": {"type": "string", "description": "What was done."}
+            },
+            "required": ["summary"],
+        },
+    },
+}
+
+# Finishing before any real work has happened would record an empty
+# milestone as a success, so the claim is refused once first.
+_NOT_YET = (
+    "You have not made any change yet, so the plan cannot be complete. Carry "
+    "out the next step of the plan, or run the tests to show the work is "
+    "already done, before calling finish."
+)
+# A reply in prose breaks the tool contract. A server that enforces
+# tool_choice never allows one; on one that doesn't, the model is told once.
+_PROSE_NUDGE = (
+    "\n\nYour last reply was prose. Every reply must be a tool call: carry "
+    "out the next step, or call finish if the plan is complete."
 )
 
 
@@ -128,8 +150,8 @@ class LLMExecutor:
         # A12: ask_expert exists in addition to the observable escalation
         # triggers, never as a replacement for them.
         if self._expert is None:
-            return EXECUTOR_TOOLS
-        return [*EXECUTOR_TOOLS, ASK_EXPERT_TOOL]
+            return [*EXECUTOR_TOOLS, FINISH_TOOL]
+        return [*EXECUTOR_TOOLS, ASK_EXPERT_TOOL, FINISH_TOOL]
 
     def _consult(self, arguments: dict[str, Any]) -> ToolOutcome:
         assert self._expert is not None
@@ -173,9 +195,12 @@ class LLMExecutor:
         if plan is not None:
             # In the system prompt, so the context budget never drops it.
             system += f"\n\nThe approved plan:\n{plan}"
+        tools = self._tools()
+        offered = {tool["function"]["name"] for tool in tools}
         consecutive_failures = 0
         did_work = False
-        challenged = False
+        finish_refused = False
+        prose_nudged = False
 
         for turn in range(self._max_turns):
             # A4: `stop` takes effect at the next tool boundary. The start of
@@ -185,7 +210,7 @@ class LLMExecutor:
             try:
                 produced = self._client.complete(
                     self._store.read_events(after=after),
-                    tools=self._tools(),
+                    tools=tools,
                     model=model,
                     system=system,
                 )
@@ -199,19 +224,12 @@ class LLMExecutor:
                     calls.append(event)
 
             if not calls:
-                if did_work:
-                    return ExecutorOutcome(
-                        "completed", "executor reported the plan done"
-                    )
-                if not challenged:
-                    # Claiming completion without having touched anything is
-                    # not evidence of completion; ask once before believing it.
-                    challenged = True
-                    system += _NUDGE
+                if not prose_nudged:
+                    prose_nudged = True
+                    system += _PROSE_NUDGE
                     continue
                 return ExecutorOutcome(
-                    "escalate",
-                    "executor claimed the milestone was done without making any change",
+                    "escalate", "executor kept replying in prose instead of tools"
                 )
 
             self._report(turn, calls)
@@ -219,11 +237,47 @@ class LLMExecutor:
                 if self._cancel.requested:
                     skip_calls(self._store, calls[index:])
                     return ExecutorOutcome("stopped", f"during turn {turn + 1}")
-                if call.tool in MUTATING_TOOLS:
-                    did_work = True
-                if call.tool == "ask_expert":
+                if call.tool == "finish":
+                    if not did_work and not finish_refused:
+                        finish_refused = True
+                        outcome = ToolOutcome(_NOT_YET, success=False)
+                    elif not did_work:
+                        return ExecutorOutcome(
+                            "escalate",
+                            "executor claimed the milestone was done without "
+                            "making any change",
+                        )
+                    else:
+                        self._store.append_event(
+                            ToolResult(
+                                call_id=call.call_id,
+                                result="milestone finished",
+                                success=True,
+                            )
+                        )
+                        skip_calls(
+                            self._store,
+                            calls[index + 1 :],
+                            "not run: finish had already ended the milestone",
+                        )
+                        summary = str(call.arguments.get("summary", "")).strip()
+                        return ExecutorOutcome(
+                            "completed", summary or "executor reported the plan done"
+                        )
+                elif call.tool not in offered:
+                    # A model inventing tools is not doing the work, and with
+                    # no server enforcement nothing else would stop it, so it
+                    # counts toward escalation like a failed command (A12).
+                    outcome = ToolOutcome(
+                        f"no such tool: {call.tool}; the tools are "
+                        + ", ".join(sorted(offered)),
+                        success=False,
+                    )
+                elif call.tool == "ask_expert":
                     outcome = self._consult(call.arguments)
                 else:
+                    if call.tool in MUTATING_TOOLS:
+                        did_work = True
                     outcome = self._workspace.dispatch(call.tool, call.arguments)
                 self._store.append_event(
                     ToolResult(
@@ -232,9 +286,10 @@ class LLMExecutor:
                         success=outcome.success,
                     )
                 )
-                # Only commands count toward escalation: a failed read is the
-                # model looking around, a failed command is work not working.
-                if call.tool == "run_command":
+                # Commands and invented tools count toward escalation: a
+                # failed read is the model looking around, a failed command
+                # is work not working.
+                if call.tool == "run_command" or call.tool not in offered:
                     consecutive_failures = (
                         0 if outcome.success else consecutive_failures + 1
                     )

nisse-dev/src/nisse_dev/llm.py

@@ -142,6 +142,7 @@ class LiteLLMClient:
         # a side channel rather than a return value, so the LLMClient
         # protocol stays the same for every implementation.
         self.last_usage: dict[str, int] = {}
+        self.last_model: str | None = None
 
     def complete(
         self,
@@ -151,8 +152,12 @@ class LiteLLMClient:
         system: str | None = None,
         *,
         max_tokens: int | None = None,
+        require_tool_call: bool = False,
     ) -> list[CanonicalEvent]:
         body = self._renderer.render(events, tools, model, system=system)
+        if require_tool_call and tools:
+            # Only for models whose server enforces it; see BudgetedClient.
+            body["tool_choice"] = "required"
         # A ceiling on this route too: without one a runaway turn is bounded
         # only by the provider's default, and the same truncation check has
         # nothing to check against. A caller that knows the model's limits
@@ -182,6 +187,9 @@ class LiteLLMClient:
                 "llm.max_output_tokens, or the model's max_output_tokens in LiteLLM"
             )
         produced_by = data.get("model") or requested_model
+        # What actually answered, which is not always what was asked for: a
+        # proxy fallback answers under another model's name.
+        self.last_model = produced_by
         usage = data.get("usage") or {}
         self.last_usage = {
             "input": int(usage.get("prompt_tokens") or 0),
@@ -254,6 +262,7 @@ class AnthropicPassthroughClient:
         self._max_tokens = max_tokens or DEFAULT_MAX_OUTPUT_TOKENS
         self._sleep = sleep
         self.last_usage: dict[str, int] = {}
+        self.last_model: str | None = None
 
     def complete(
         self,
@@ -263,6 +272,7 @@ class AnthropicPassthroughClient:
         system: str | None = None,
         *,
         max_tokens: int | None = None,
+        require_tool_call: bool = False,
     ) -> list[CanonicalEvent]:
         body = self._renderer.render(
             events,
@@ -271,6 +281,8 @@ class AnthropicPassthroughClient:
             system=system,
             max_tokens=max_tokens or self._max_tokens,
         )
+        if require_tool_call and tools:
+            body["tool_choice"] = {"type": "any"}  # Anthropic's name for it
         response = _post_with_rate_limit_retry(
             self._http, "/v1/messages", body, self._sleep
         )
@@ -293,6 +305,9 @@ class AnthropicPassthroughClient:
         if not isinstance(content, list):
             raise LLMError(f"malformed passthrough response: {str(data)[:300]}")
         produced_by = data.get("model") or requested_model
+        # What actually answered, which is not always what was asked for: a
+        # proxy fallback answers under another model's name.
+        self.last_model = produced_by
         usage = data.get("usage") or {}
         self.last_usage = {
             "input": int(usage.get("input_tokens") or 0),

nisse-dev/src/nisse_dev/probe.py

@@ -11,6 +11,10 @@ with it:
 - **It took the whole prompt.** The server allows more than is declared, so
   the declaration is conservative.
 
+Every answer is checked for which model gave it. A proxy that falls back to
+another model on an error returns a normal-looking answer under that model's
+name, and once made this probe report a refusing server as roomy.
+
 Run it with ``scripts/probe_context.py``. It spends one small and one large
 request on the model, so it refuses windows above a ceiling unless told
 otherwise: commercial models' limits are already known to LiteLLM.
@@ -20,6 +24,8 @@ from dataclasses import dataclass
 
 import httpx
 
+from nisse_dev.catalog import ModelFacts
+
 # Of a probe's size, how much must come back for it to count as uncut. A
 # chat template adds a little, and a tokenizer counts a little differently.
 _UNCUT = 0.9
@@ -86,7 +92,14 @@ def _prompt(words: int) -> str:
     return " ".join(f"item{i}" for i in range(words)) + "\n\nReply with just: ok"
 
 
-def _send(http: httpx.Client, model: str, words: int) -> tuple[int | None, str | None]:
+class Fallback(Exception):
+    def __init__(self, answered_by: str) -> None:
+        self.answered_by = answered_by
+
+
+def _send(
+    http: httpx.Client, model: str, words: int, backing: str
+) -> tuple[int | None, str | None]:
     response = http.post(
         "/chat/completions",
         json={
@@ -97,18 +110,33 @@ def _send(http: httpx.Client, model: str, words: int) -> tuple[int | None, str |
     )
     if response.status_code != 200:
         return None, f"{response.status_code}: {response.text}"
-    usage = response.json().get("usage") or {}
+    data = response.json()
+    answered = str(data.get("model") or "")
+    facts = ModelFacts(alias=model, backing=backing, provider="", supports_tools=None)
+    if not facts.answered(answered):
+        raise Fallback(answered)
+    usage = data.get("usage") or {}
     return int(usage.get("prompt_tokens") or 0), None
 
 
-def probe(http: httpx.Client, model: str, declared: int) -> ProbeResult:
+def probe(
+    http: httpx.Client, model: str, declared: int, backing: str = ""
+) -> ProbeResult:
     """Measure how many tokens a word of the probe costs on this model, then
     send a prompt half again as long as the declared window."""
     calibration_words = 200
-    tokens, error = _send(http, model, calibration_words)
-    if error is not None or not tokens:
-        return ProbeResult("failed", f"a short calibration request failed: {error}")
-    per_word = tokens / calibration_words
-    sent = int(declared * 1.5)
-    reported, error = _send(http, model, int(sent / per_word))
+    try:
+        tokens, error = _send(http, model, calibration_words, backing)
+        if error is not None or not tokens:
+            return ProbeResult("failed", f"a short calibration request failed: {error}")
+        per_word = tokens / calibration_words
+        sent = int(declared * 1.5)
+        reported, error = _send(http, model, int(sent / per_word), backing)
+    except Fallback as fallback:
+        return ProbeResult(
+            "fallback",
+            f"{fallback.answered_by} answered instead of {model}: the proxy fell "
+            "back to another model, so nothing here describes this one. Remove "
+            "the fallback, or probe the server directly.",
+        )
     return interpret(declared, sent, reported, error)