Make the config file the only home for settings
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 51s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 51s
With no production instance yet, there is no reason to carry two ways
of configuring the agent. Every environment variable that duplicated a
config key is gone; the environment now supplies only the secrets the
file references by ${…}, plus NISSE_CONFIG and NISSE_ENV to find the
files.
- AgentConfig loads from a file (load, from_file) or a parsed tree
(from_tree, for tests); a missing file is an error naming the example
- The from_env constructors on LiteLLMClient, AnthropicPassthroughClient,
GiteaClient, MatrixConfig and RoleMap are removed, with the env
reading for max output tokens; clients take their settings from config
- The live integration tests share a committed agent config,
tests/integration/nisse-test.yaml, aimed at test resources, and one
conftest that loads it and builds clients. NISSE_LIVE_PLANNER_MODEL
becomes its planner role; NISSE_CAPABILITY_MODELS stays as a per-run
switch
- A missing secret now fails the live tests instead of skipping them
- CI passes only the three tokens
- .env.example is secrets only; spec A5 says config comes from the file
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
df449c7e34
commit
15d69799bf
28 files changed
+571
-777
No files matched your search
@@ -14,7 +14,6 @@ because it spends real tokens across several models.
|
||||
"""
|
||||
|
||||
import os
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -26,14 +25,12 @@ from nisse_dev.canonical import (
|
||||
UserMessage,
|
||||
)
|
||||
from nisse_dev.catalog import ModelCatalog, select_client
|
||||
from nisse_dev.envfile import load_env_file
|
||||
from nisse_dev.config import AgentConfig
|
||||
from nisse_dev.expert import parse_answer
|
||||
from nisse_dev.llm import AnthropicPassthroughClient, LiteLLMClient, LLMError
|
||||
from nisse_dev.planner import PLANNER_TOOLS
|
||||
from nisse_dev.tools import EXECUTOR_TOOLS
|
||||
|
||||
load_env_file(Path(__file__).resolve().parents[2] / ".env")
|
||||
|
||||
SENTINEL = "NISSE_CAPABILITY_MODELS"
|
||||
MODELS = [m.strip() for m in os.environ.get(SENTINEL, "").split(",") if m.strip()]
|
||||
|
||||
@@ -45,17 +42,18 @@ pytestmark = [
|
||||
]
|
||||
|
||||
|
||||
def clients() -> tuple[LiteLLMClient, AnthropicPassthroughClient]:
|
||||
return LiteLLMClient.from_env(), AnthropicPassthroughClient.from_env()
|
||||
@pytest.fixture
|
||||
def routes(
|
||||
litellm: LiteLLMClient, passthrough: AnthropicPassthroughClient
|
||||
) -> tuple[LiteLLMClient, AnthropicPassthroughClient]:
|
||||
return litellm, passthrough
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def catalog() -> ModelCatalog:
|
||||
def catalog(live_config: AgentConfig) -> ModelCatalog:
|
||||
"""The live catalog this key can see. /model/info is scoped per key, so
|
||||
the probe list and the key have to agree."""
|
||||
return ModelCatalog.discover(
|
||||
os.environ["NISSE_LLM_BASE_URL"], os.environ["NISSE_LLM_TOKEN"]
|
||||
)
|
||||
return ModelCatalog.discover(live_config.llm_base_url, live_config.llm_token)
|
||||
|
||||
|
||||
def user(text: str) -> list[StoredEvent]:
|
||||
@@ -63,8 +61,8 @@ def user(text: str) -> list[StoredEvent]:
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_model_answers_at_all(model: str, catalog: ModelCatalog) -> None:
|
||||
normalized, passthrough = clients()
|
||||
def test_model_answers_at_all(model: str, catalog: ModelCatalog, routes) -> None:
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
produced = client.complete(
|
||||
user("Reply with exactly one word: pong"), None, model, system="Be terse."
|
||||
@@ -74,10 +72,12 @@ def test_model_answers_at_all(model: str, catalog: ModelCatalog) -> None:
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_model_holds_the_planner_contract(model: str, catalog: ModelCatalog) -> None:
|
||||
def test_model_holds_the_planner_contract(
|
||||
model: str, catalog: ModelCatalog, routes
|
||||
) -> None:
|
||||
"""A4's gating is structural at the tool layer: a model that answers in
|
||||
prose cannot be gated, because there is no plan to approve."""
|
||||
normalized, passthrough = clients()
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
produced = client.complete(
|
||||
user("Draft an architecture plan for a command-line slug generator."),
|
||||
@@ -101,10 +101,12 @@ def test_model_holds_the_planner_contract(model: str, catalog: ModelCatalog) ->
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_model_round_trips_a_tool_result(model: str, catalog: ModelCatalog) -> None:
|
||||
def test_model_round_trips_a_tool_result(
|
||||
model: str, catalog: ModelCatalog, routes
|
||||
) -> None:
|
||||
"""The executor loop feeds results back; a model that cannot consume its
|
||||
own tool result cannot run a milestone."""
|
||||
normalized, passthrough = clients()
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
events = user("List the files in the current directory using your tools.")
|
||||
produced = client.complete(events, EXECUTOR_TOOLS, model, system="Use a tool.")
|
||||
@@ -130,11 +132,11 @@ def test_model_round_trips_a_tool_result(model: str, catalog: ModelCatalog) -> N
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_model_holds_the_prompt_level_json_contract(
|
||||
model: str, catalog: ModelCatalog
|
||||
model: str, catalog: ModelCatalog, routes
|
||||
) -> None:
|
||||
"""A14 contracts structured output at the prompt level because vendors
|
||||
differ on JSON enforcement. This measures whether that holds per model."""
|
||||
normalized, passthrough = clients()
|
||||
normalized, passthrough = routes
|
||||
client = select_client(model, normalized, passthrough, catalog)
|
||||
produced = client.complete(
|
||||
user("Is SQLite sufficient for a single-process agent's task state?"),
|
||||
@@ -153,11 +155,13 @@ def test_model_holds_the_prompt_level_json_contract(
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", MODELS)
|
||||
def test_route_selection_matches_the_model(model: str, catalog: ModelCatalog) -> None:
|
||||
def test_route_selection_matches_the_model(
|
||||
model: str, catalog: ModelCatalog, routes
|
||||
) -> None:
|
||||
"""A non-Anthropic model must not be sent Anthropic-shaped bodies. This
|
||||
was hardcoded until gpt-6-astra made it observable, and name-sniffed until
|
||||
semantic aliases made the name stop meaning anything."""
|
||||
normalized, passthrough = clients()
|
||||
normalized, passthrough = routes
|
||||
facts = catalog.facts(model)
|
||||
chosen = select_client(model, normalized, passthrough, catalog)
|
||||
if facts.provider == "anthropic":
|
||||
|
||||
Reference in new issue
Block a user