diff --git a/.env.example b/.env.example index 4052e03..fc774ee 100644 --- a/.env.example +++ b/.env.example @@ -12,7 +12,3 @@ NISSE_GITEA_TOKEN=your-scoped-service-account-token # Where the config file is, if not ./nisse-dev.yaml. Where this file is, if # not ./.env, is NISSE_ENV, which has to be set in the real environment. # NISSE_CONFIG=~/nisse-demo-workspace/nisse-dev.yaml - -# Opt-in for the capability matrix: models to measure, comma separated. -# Spends real tokens across every model listed. See docs/Tuning-Guide.md. -# NISSE_CAPABILITY_MODELS=local-fast,local-coder diff --git a/docs/Tuning-Guide.md b/docs/Tuning-Guide.md index ef49dea..49fb34f 100644 --- a/docs/Tuning-Guide.md +++ b/docs/Tuning-Guide.md @@ -248,8 +248,8 @@ change one thing at a time. 1. **Model and quantization.** Start with whether a candidate holds the tool-call contract at all: - NISSE_CAPABILITY_MODELS=alias-a,alias-b \ - uv run pytest -m integration tests/integration/test_model_capability.py + uv run pytest -m integration tests/integration/test_model_capability.py \ + --capability-models=alias-a,alias-b A model that falls back to prose instead of calling tools is disqualified, however good its benchmarks look. Before comparing aliases, check what diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 0000000..a7d20d5 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,20 @@ +"""Command-line options for the repository-level test suites. + +Declared here rather than beside the tests that use them: pytest only reads +options from conftest files it loads at startup, and this directory is one +of the configured test paths. +""" + +import pytest + + +def pytest_addoption(parser: pytest.Parser) -> None: + parser.addoption( + "--capability-models", + default="", + help=( + "comma-separated LiteLLM aliases to measure with the capability " + "matrix (tests/integration/test_model_capability.py). Spends real " + "tokens on every model listed; the matrix skips when this is empty." + ), + ) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 7a983fa..67c2866 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -21,8 +21,8 @@ from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient REPO = Path(__file__).resolve().parents[2] TEST_CONFIG = Path(__file__).parent / "nisse-test.yaml" -# At import, so per-run switches such as NISSE_CAPABILITY_MODELS can be set -# in .env too. Real environment variables win over the file. +# The secrets nisse-test.yaml refers to, for local runs. CI sets them as real +# environment variables, which win over the file. load_env_file(REPO / ".env") diff --git a/tests/integration/test_model_capability.py b/tests/integration/test_model_capability.py index 149e352..8205cf5 100644 --- a/tests/integration/test_model_capability.py +++ b/tests/integration/test_model_capability.py @@ -9,11 +9,15 @@ What it measures is the thing that actually breaks: whether a model can hold the tool-call contract our loops depend on. llama3.2 could not — it returned `{` as an entire plan — and no amount of prompt work fixed it. -Opt in with NISSE_CAPABILITY_MODELS, comma separated. Skipped otherwise, -because it spends real tokens across several models. -""" +Opt in by naming the models to measure, for one run: -import os + uv run pytest -m integration tests/integration/test_model_capability.py \ + --capability-models=local-fast,local-coder + +Skipped otherwise, because it spends real tokens across every model listed. +Which models to measure is a question for a particular run, so it is an +argument, not configuration. +""" import pytest @@ -31,15 +35,28 @@ from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError from nisse_dev.planner import PLANNER_TOOLS from nisse_dev.tools import EXECUTOR_TOOLS -SENTINEL = "NISSE_CAPABILITY_MODELS" -MODELS = [m.strip() for m in os.environ.get(SENTINEL, "").split(",") if m.strip()] +pytestmark = pytest.mark.integration -pytestmark = [ - pytest.mark.integration, - pytest.mark.skipif( - not MODELS, reason=f"set {SENTINEL} to a comma-separated model list" - ), -] + +def pytest_generate_tests(metafunc: pytest.Metafunc) -> None: + if "model" not in metafunc.fixturenames: + return + raw = metafunc.config.getoption("--capability-models") + models = [m.strip() for m in raw.split(",") if m.strip()] + if models: + metafunc.parametrize("model", models) + else: + metafunc.parametrize( + "model", + [ + pytest.param( + None, + marks=pytest.mark.skip( + reason="no models to measure; pass --capability-models" + ), + ) + ], + ) @pytest.fixture @@ -60,7 +77,6 @@ def user(text: str) -> list[StoredEvent]: return [StoredEvent(id=1, ts="t", event=UserMessage(text=text, source="probe"))] -@pytest.mark.parametrize("model", MODELS) def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None: normalized, passthrough = routes client = select_client(model, normalized, passthrough, catalog) @@ -71,7 +87,6 @@ def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None assert texts, f"{model} produced no text: {produced!r}" -@pytest.mark.parametrize("model", MODELS) def test_model_holds_the_planner_contract( model: str, catalog: ModelCatalog, routes ) -> None: @@ -100,7 +115,6 @@ def test_model_holds_the_planner_contract( ) -@pytest.mark.parametrize("model", MODELS) def test_model_round_trips_a_tool_result( model: str, catalog: ModelCatalog, routes ) -> None: @@ -130,7 +144,6 @@ def test_model_round_trips_a_tool_result( assert follow_up, f"{model} produced nothing after its tool result" -@pytest.mark.parametrize("model", MODELS) def test_model_holds_the_prompt_level_json_contract( model: str, catalog: ModelCatalog, routes ) -> None: @@ -154,7 +167,6 @@ def test_model_holds_the_prompt_level_json_contract( ) -@pytest.mark.parametrize("model", MODELS) def test_route_selection_matches_the_model( model: str, catalog: ModelCatalog, routes ) -> None: