dongarandClaude Opus 5.5
e351d0a74e
A pure move; behaviour unchanged. - models (L9, L7): llm, catalog, probe, render, render_anthropic, context - log (L7, L6): canonical, views - state (L8): store (was state), reconcile, worker, cancel, dump - knowledge (L10, L6.4): vault, memory - conversation (L5): transport, matrix - execution (L13): ssh (was execution), testing - authority (P3, P4, L4): actions - gitea, oplog, envfile stay at the top until each gains a second module. Every import, monkeypatch target, script and doc reference rewritten. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
56 lines
1.8 KiB
Python
56 lines
1.8 KiB
Python
"""The action classifier against its evaluation set (L4, M3.2), per model.
|
|
|
|
Opt in by naming the models, as for the capability matrix:
|
|
|
|
uv run pytest -m integration tests/integration/test_classifier_cases.py \
|
|
--capability-models=qwen3.8-27b-nothink
|
|
|
|
Skipped otherwise. Each case passes when the model's verdict is one of those
|
|
the case allows; a case that fails is a model to watch, or a case to rethink.
|
|
"""
|
|
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
import pytest
|
|
import yaml
|
|
|
|
from nisse_core.authority.actions import Action
|
|
from nisse_core.models.llm import LiteLLMClient
|
|
from nisse_dev.loop.classifier import ActionClassifier, Scope
|
|
|
|
pytestmark = pytest.mark.integration
|
|
|
|
CASES: list[dict[str, Any]] = yaml.safe_load(
|
|
(Path(__file__).parent / "classifier_cases.yaml").read_text()
|
|
)
|
|
|
|
|
|
def pytest_generate_tests(metafunc: pytest.Metafunc) -> None:
|
|
if "model" not in metafunc.fixturenames:
|
|
return
|
|
raw = metafunc.config.getoption("--capability-models")
|
|
models = [m.strip() for m in raw.split(",") if m.strip()]
|
|
if not models:
|
|
metafunc.parametrize(
|
|
"model",
|
|
[
|
|
pytest.param(
|
|
None, marks=pytest.mark.skip(reason="pass --capability-models")
|
|
)
|
|
],
|
|
)
|
|
return
|
|
metafunc.parametrize("model", models)
|
|
|
|
|
|
@pytest.mark.parametrize("case", CASES, ids=[c["name"] for c in CASES])
|
|
def test_the_classifier_judges_the_case(
|
|
model: str, case: dict[str, Any], litellm: LiteLLMClient
|
|
) -> None:
|
|
classifier = ActionClassifier(litellm, model)
|
|
scope = Scope(project="demo", repo="dev/demo", issue=7, agreed=case["agreed"])
|
|
action = Action("run_command", {"command": case["command"]})
|
|
verdict, _ = classifier.judge(action, scope, "/srv/work/demo")
|
|
assert verdict.rule in case["expect"], f"{model}: {verdict.rule}: {verdict.reason}"
|