.env.example

@@ -12,7 +12,3 @@ NISSE_GITEA_TOKEN=your-scoped-service-account-token
 # Where the config file is, if not ./nisse-dev.yaml. Where this file is, if
 # not ./.env, is NISSE_ENV, which has to be set in the real environment.
 # NISSE_CONFIG=~/nisse-demo-workspace/nisse-dev.yaml
-
-# Opt-in for the capability matrix: models to measure, comma separated.
-# Spends real tokens across every model listed. See docs/Tuning-Guide.md.
-# NISSE_CAPABILITY_MODELS=local-fast,local-coder

docs/Tuning-Guide.md

@@ -248,8 +248,8 @@ change one thing at a time.
 1. **Model and quantization.** Start with whether a candidate holds the
    tool-call contract at all:
 
-       NISSE_CAPABILITY_MODELS=alias-a,alias-b \
-         uv run pytest -m integration tests/integration/test_model_capability.py
+       uv run pytest -m integration tests/integration/test_model_capability.py \
+         --capability-models=alias-a,alias-b
 
    A model that falls back to prose instead of calling tools is disqualified,
    however good its benchmarks look. Before comparing aliases, check what

tests/conftest.py

@@ -0,0 +1,20 @@
+"""Command-line options for the repository-level test suites.
+
+Declared here rather than beside the tests that use them: pytest only reads
+options from conftest files it loads at startup, and this directory is one
+of the configured test paths.
+"""
+
+import pytest
+
+
+def pytest_addoption(parser: pytest.Parser) -> None:
+    parser.addoption(
+        "--capability-models",
+        default="",
+        help=(
+            "comma-separated LiteLLM aliases to measure with the capability "
+            "matrix (tests/integration/test_model_capability.py). Spends real "
+            "tokens on every model listed; the matrix skips when this is empty."
+        ),
+    )

tests/integration/conftest.py

@@ -21,8 +21,8 @@ from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient
 REPO = Path(__file__).resolve().parents[2]
 TEST_CONFIG = Path(__file__).parent / "nisse-test.yaml"
 
-# At import, so per-run switches such as NISSE_CAPABILITY_MODELS can be set
-# in .env too. Real environment variables win over the file.
+# The secrets nisse-test.yaml refers to, for local runs. CI sets them as real
+# environment variables, which win over the file.
 load_env_file(REPO / ".env")
 
 

tests/integration/test_model_capability.py

@@ -9,11 +9,15 @@ What it measures is the thing that actually breaks: whether a model can hold
 the tool-call contract our loops depend on. llama3.2 could not — it returned
 `{` as an entire plan — and no amount of prompt work fixed it.
 
-Opt in with NISSE_CAPABILITY_MODELS, comma separated. Skipped otherwise,
-because it spends real tokens across several models.
-"""
+Opt in by naming the models to measure, for one run:
 
-import os
+    uv run pytest -m integration tests/integration/test_model_capability.py \
+        --capability-models=local-fast,local-coder
+
+Skipped otherwise, because it spends real tokens across every model listed.
+Which models to measure is a question for a particular run, so it is an
+argument, not configuration.
+"""
 
 import pytest
 
@@ -31,15 +35,28 @@ from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError
 from nisse_dev.planner import PLANNER_TOOLS
 from nisse_dev.tools import EXECUTOR_TOOLS
 
-SENTINEL = "NISSE_CAPABILITY_MODELS"
-MODELS = [m.strip() for m in os.environ.get(SENTINEL, "").split(",") if m.strip()]
+pytestmark = pytest.mark.integration
 
-pytestmark = [
-    pytest.mark.integration,
-    pytest.mark.skipif(
-        not MODELS, reason=f"set {SENTINEL} to a comma-separated model list"
-    ),
-]
+
+def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
+    if "model" not in metafunc.fixturenames:
+        return
+    raw = metafunc.config.getoption("--capability-models")
+    models = [m.strip() for m in raw.split(",") if m.strip()]
+    if models:
+        metafunc.parametrize("model", models)
+    else:
+        metafunc.parametrize(
+            "model",
+            [
+                pytest.param(
+                    None,
+                    marks=pytest.mark.skip(
+                        reason="no models to measure; pass --capability-models"
+                    ),
+                )
+            ],
+        )
 
 
 @pytest.fixture
@@ -60,7 +77,6 @@ def user(text: str) -> list[StoredEvent]:
     return [StoredEvent(id=1, ts="t", event=UserMessage(text=text, source="probe"))]
 
 
-@pytest.mark.parametrize("model", MODELS)
 def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None:
     normalized, passthrough = routes
     client = select_client(model, normalized, passthrough, catalog)
@@ -71,7 +87,6 @@ def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None
     assert texts, f"{model} produced no text: {produced!r}"
 
 
-@pytest.mark.parametrize("model", MODELS)
 def test_model_holds_the_planner_contract(
     model: str, catalog: ModelCatalog, routes
 ) -> None:
@@ -100,7 +115,6 @@ def test_model_holds_the_planner_contract(
         )
 
 
-@pytest.mark.parametrize("model", MODELS)
 def test_model_round_trips_a_tool_result(
     model: str, catalog: ModelCatalog, routes
 ) -> None:
@@ -130,7 +144,6 @@ def test_model_round_trips_a_tool_result(
     assert follow_up, f"{model} produced nothing after its tool result"
 
 
-@pytest.mark.parametrize("model", MODELS)
 def test_model_holds_the_prompt_level_json_contract(
     model: str, catalog: ModelCatalog, routes
 ) -> None:
@@ -154,7 +167,6 @@ def test_model_holds_the_prompt_level_json_contract(
     )
 
 
-@pytest.mark.parametrize("model", MODELS)
 def test_route_selection_matches_the_model(
     model: str, catalog: ModelCatalog, routes
 ) -> None: