Enforce the tool-call contract, asking servers only where it works

The executor ended a milestone on any reply without a tool call, and
nothing held a model to the offered tools. Measured on the 4070:
tool_choice "required" is enforced by vLLM (local-fast) and Anthropic,
but qwen3-14b (local-coder) under it cannot stop and writes until the
output ceiling, a known model bug. So enforcement is asked for per
model, and the loop holds the contract itself everywhere.

- The catalog reads supports_tool_choice from LiteLLM's model_info;
  BudgetedClient sends tool_choice only to models declared true, as
  "required" on the normalized route and {"type": "any"} on the
  Anthropic passthrough, and only when tools are offered
- The executor gains an explicit finish tool, the only way to end a
  milestone. Finishing with nothing changed is refused once; a prose
  reply gets one nudge, then escalates; a call to a tool not offered
  counts toward failure_limit, like a failed command; calls after
  finish are recorded as not run
- Clients record the model that actually answered. ModelFacts.answered
  recognises a proxy fallback, which once credited Claude's answers to
  a local model that could not call tools at all
- The context probe reports a fallback instead of trusting it; rerun,
  both local models now refuse over-long prompts, as they should
- The capability matrix checks every answer came from the model asked,
  that models call only offered tools, and that a declared
  supports_tool_choice is true
- Spec A15 and the tuning guide say how enforcement is requested,
  declared and backed up

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

tests/integration/test_model_capability.py

@@ -30,6 +30,7 @@ from nisse_dev.canonical import (
 )
 from nisse_dev.catalog import ModelCatalog, select_client
 from nisse_dev.config import AgentConfig
+from nisse_dev.executor import FINISH_TOOL
 from nisse_dev.expert import parse_answer
 from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError
 from nisse_dev.planner import PLANNER_TOOLS
@@ -73,6 +74,16 @@ def catalog(live_config: AgentConfig) -> ModelCatalog:
     return ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token)
 
 
+def assert_answered(client: object, model: str, catalog: ModelCatalog) -> None:
+    """A proxy fallback answers under another model's name. Crediting its
+    answer to this model is how a local model once 'held the tool contract
+    20/20' while never answering at all."""
+    reported = getattr(client, "last_model", None)
+    assert catalog.facts(model).answered(reported), (
+        f"{reported} answered instead of {model}: a fallback, not a measurement"
+    )
+
+
 def user(text: str) -> list[StoredEvent]:
     return [StoredEvent(id=1, ts="t", event=UserMessage(text=text, source="probe"))]
 
@@ -83,6 +94,7 @@ def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None
     produced = client.complete(
         user("Reply with exactly one word: pong"), None, model, system="Be terse."
     )
+    assert_answered(client, model, catalog)
     texts = [e.text for e in produced if isinstance(e, AssistantText)]
     assert texts, f"{model} produced no text: {produced!r}"
 
@@ -103,6 +115,7 @@ def test_model_holds_the_planner_contract(
             "ask_human. Plain prose is not accepted."
         ),
     )
+    assert_answered(client, model, catalog)
     calls = [e for e in produced if isinstance(e, ToolCall)]
     assert calls, f"{model} answered in prose instead of calling a tool"
     assert calls[0].tool in {"submit_plan", "ask_human"}
@@ -140,6 +153,7 @@ def test_model_round_trips_a_tool_result(
             ),
         )
     )
+    assert_answered(client, model, catalog)
     follow_up = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.")
     assert follow_up, f"{model} produced nothing after its tool result"
 
@@ -161,6 +175,7 @@ def test_model_holds_the_prompt_level_json_contract(
             '"caveats": "..."}'
         ),
     )
+    assert_answered(client, model, catalog)
     text = "\n".join(e.text for e in produced if isinstance(e, AssistantText))
     assert parse_answer(text) is not None, (
         f"{model} did not produce parseable JSON: {text[:200]!r}"
@@ -185,3 +200,59 @@ def test_route_selection_matches_the_model(
         chosen.complete(user("say pong"), None, model, system="Be terse.")
     except LLMError as exc:
         pytest.fail(f"{model} failed on its selected route: {exc}")
+    assert_answered(chosen, model, catalog)
+
+
+FINISH_TOOLS = [*EXECUTOR_TOOLS, FINISH_TOOL]
+OFFERED = {tool["function"]["name"] for tool in FINISH_TOOLS}
+
+
+def test_model_calls_only_the_tools_it_was_offered(
+    model: str, catalog: ModelCatalog, routes
+) -> None:
+    """Measured on a server that did not enforce tool_choice: a model asked
+    to chat called "chat", "say_hello" and "respond", none of them offered.
+    The executor counts those as failures; this measures how often."""
+    normalized, passthrough = routes
+    client = select_client(model, normalized, passthrough, catalog)
+    produced = client.complete(
+        user("Hi! How are you today? Just chat with me for a moment."),
+        FINISH_TOOLS,
+        model,
+        system="You are an agent. Every reply must be a tool call.",
+    )
+    assert_answered(client, model, catalog)
+    called = {e.tool for e in produced if isinstance(e, ToolCall)}
+    assert called <= OFFERED, (
+        f"{model} called tools it was never offered: {called - OFFERED}"
+    )
+
+
+def test_supports_tool_choice_is_declared_truthfully(
+    model: str, catalog: ModelCatalog, routes
+) -> None:
+    """The agent asks for tool_choice only of models LiteLLM declares
+    supports_tool_choice for, so the declaration must be true. Asked of a
+    model that can't honour it, "required" can be worse than not asking:
+    qwen3-14b can't stop, and writes until the output ceiling."""
+    facts = catalog.facts(model)
+    if facts.supports_tool_choice is not True:
+        pytest.skip(
+            f"{model} is not declared to support tool_choice, so is never asked"
+        )
+    normalized, passthrough = routes
+    client = select_client(model, normalized, passthrough, catalog)
+    produced = client.complete(
+        user("Hi! How are you today? Just chat with me for a moment."),
+        FINISH_TOOLS,
+        model,
+        system="You are a helpful assistant.",
+        require_tool_call=True,
+    )
+    assert_answered(client, model, catalog)
+    calls = [e for e in produced if isinstance(e, ToolCall)]
+    assert calls, (
+        f"{model} is declared supports_tool_choice but answered a required call "
+        "in prose; set it to false in LiteLLM"
+    )
+    assert {c.tool for c in calls} <= OFFERED