Enforce the tool-call contract, asking servers only where it works
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 56s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 56s
The executor ended a milestone on any reply without a tool call, and
nothing held a model to the offered tools. Measured on the 4070:
tool_choice "required" is enforced by vLLM (local-fast) and Anthropic,
but qwen3-14b (local-coder) under it cannot stop and writes until the
output ceiling, a known model bug. So enforcement is asked for per
model, and the loop holds the contract itself everywhere.
- The catalog reads supports_tool_choice from LiteLLM's model_info;
BudgetedClient sends tool_choice only to models declared true, as
"required" on the normalized route and {"type": "any"} on the
Anthropic passthrough, and only when tools are offered
- The executor gains an explicit finish tool, the only way to end a
milestone. Finishing with nothing changed is refused once; a prose
reply gets one nudge, then escalates; a call to a tool not offered
counts toward failure_limit, like a failed command; calls after
finish are recorded as not run
- Clients record the model that actually answered. ModelFacts.answered
recognises a proxy fallback, which once credited Claude's answers to
a local model that could not call tools at all
- The context probe reports a fallback instead of trusting it; rerun,
both local models now refuse over-long prompts, as they should
- The capability matrix checks every answer came from the model asked,
that models call only offered tools, and that a declared
supports_tool_choice is true
- Spec A15 and the tuning guide say how enforcement is requested,
declared and backed up
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
389b71a5d1
commit
b934c46714
16 files changed
+568
-86
No files matched your search
@@ -1,5 +1,6 @@
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
@@ -341,3 +342,73 @@ def test_budget_refusal_is_a_kind_of_upstream_trouble() -> None:
|
||||
# So one catch covers the family, but it can still be told apart where
|
||||
# the difference matters — it will not clear within the hour.
|
||||
assert issubclass(BudgetExhausted, UpstreamUnavailable)
|
||||
|
||||
|
||||
# --- tool_choice (A15: constrained decoding) --------------------------------
|
||||
|
||||
TOOLS = [
|
||||
{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "finish",
|
||||
"description": "done",
|
||||
"parameters": {"type": "object", "properties": {}},
|
||||
},
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
def capture_bodies() -> tuple[list[dict], Any]:
|
||||
bodies: list[dict] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
bodies.append(json.loads(request.content))
|
||||
return completion({"role": "assistant", "content": "ok"})
|
||||
|
||||
return bodies, handler
|
||||
|
||||
|
||||
def test_required_is_sent_only_when_asked_and_tools_are_offered() -> None:
|
||||
bodies, handler = capture_bodies()
|
||||
client = make_client(handler)
|
||||
client.complete(EVENTS, tools=TOOLS, model="m", require_tool_call=True)
|
||||
client.complete(EVENTS, tools=TOOLS, model="m")
|
||||
client.complete(EVENTS, tools=None, model="m", require_tool_call=True)
|
||||
assert bodies[0]["tool_choice"] == "required"
|
||||
assert "tool_choice" not in bodies[1] # the server's default, auto
|
||||
assert "tool_choice" not in bodies[2] # nothing to choose from
|
||||
|
||||
|
||||
def test_the_passthrough_asks_in_anthropics_terms() -> None:
|
||||
bodies: list[dict] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
bodies.append(json.loads(request.content))
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"model": "claude-haiku-4-5",
|
||||
"stop_reason": "tool_use",
|
||||
"content": [
|
||||
{"type": "tool_use", "id": "t", "name": "finish", "input": {}}
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
client = AnthropicPassthroughClient(
|
||||
"https://litellm.test", "tok", transport=httpx.MockTransport(handler)
|
||||
)
|
||||
client.complete(
|
||||
EVENTS, tools=TOOLS, model="claude-haiku-4-5", require_tool_call=True
|
||||
)
|
||||
assert bodies[0]["tool_choice"] == {"type": "any"}
|
||||
|
||||
|
||||
def test_the_answering_model_is_recorded() -> None:
|
||||
# A proxy fallback answers under another model's name; this is how a
|
||||
# caller can tell.
|
||||
client = make_client(
|
||||
lambda request: completion({"role": "assistant", "content": "hi"})
|
||||
)
|
||||
client.complete(EVENTS, tools=None, model="local-fast")
|
||||
assert client.last_model == "gpt-test-1"
|
||||
Reference in new issue
Block a user