Pass the capability matrix's models as a pytest option
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 51s

NISSE_CAPABILITY_MODELS was neither a secret nor configuration: which
models to measure is a question for one run, so it is now an argument,
--capability-models=a,b, declared in tests/conftest.py where pytest
reads options at startup. With no models the matrix skips as before, so
CI is unchanged. The last non-secret NISSE_* variable other than
NISSE_CONFIG and NISSE_ENV is gone.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-09-23 23:32:44 +00:00
1 parent 15d69799bf
commit 58b439088f
5 files changed
+53 -25

No files matched your search

+29 -17
View File
@@ -9,11 +9,15 @@ What it measures is the thing that actually breaks: whether a model can hold
the tool-call contract our loops depend on. llama3.2 could not — it returned
`{` as an entire plan — and no amount of prompt work fixed it.
Opt in with NISSE_CAPABILITY_MODELS, comma separated. Skipped otherwise,
because it spends real tokens across several models.
"""
Opt in by naming the models to measure, for one run:
import os
uv run pytest -m integration tests/integration/test_model_capability.py \
--capability-models=local-fast,local-coder
Skipped otherwise, because it spends real tokens across every model listed.
Which models to measure is a question for a particular run, so it is an
argument, not configuration.
"""
import pytest
@@ -31,15 +35,28 @@ from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError
from nisse_dev.planner import PLANNER_TOOLS
from nisse_dev.tools import EXECUTOR_TOOLS
SENTINEL = "NISSE_CAPABILITY_MODELS"
MODELS = [m.strip() for m in os.environ.get(SENTINEL, "").split(",") if m.strip()]
pytestmark = pytest.mark.integration
pytestmark = [
pytest.mark.integration,
pytest.mark.skipif(
not MODELS, reason=f"set {SENTINEL} to a comma-separated model list"
),
]
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
if "model" not in metafunc.fixturenames:
return
raw = metafunc.config.getoption("--capability-models")
models = [m.strip() for m in raw.split(",") if m.strip()]
if models:
metafunc.parametrize("model", models)
else:
metafunc.parametrize(
"model",
[
pytest.param(
None,
marks=pytest.mark.skip(
reason="no models to measure; pass --capability-models"
),
)
],
)
@pytest.fixture
@@ -60,7 +77,6 @@ def user(text: str) -> list[StoredEvent]:
return [StoredEvent(id=1, ts="t", event=UserMessage(text=text, source="probe"))]
@pytest.mark.parametrize("model", MODELS)
def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None:
normalized, passthrough = routes
client = select_client(model, normalized, passthrough, catalog)
@@ -71,7 +87,6 @@ def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None
assert texts, f"{model} produced no text: {produced!r}"
@pytest.mark.parametrize("model", MODELS)
def test_model_holds_the_planner_contract(
model: str, catalog: ModelCatalog, routes
) -> None:
@@ -100,7 +115,6 @@ def test_model_holds_the_planner_contract(
)
@pytest.mark.parametrize("model", MODELS)
def test_model_round_trips_a_tool_result(
model: str, catalog: ModelCatalog, routes
) -> None:
@@ -130,7 +144,6 @@ def test_model_round_trips_a_tool_result(
assert follow_up, f"{model} produced nothing after its tool result"
@pytest.mark.parametrize("model", MODELS)
def test_model_holds_the_prompt_level_json_contract(
model: str, catalog: ModelCatalog, routes
) -> None:
@@ -154,7 +167,6 @@ def test_model_holds_the_prompt_level_json_contract(
)
@pytest.mark.parametrize("model", MODELS)
def test_route_selection_matches_the_model(
model: str, catalog: ModelCatalog, routes
) -> None: