Move configuration to a YAML file, and make the tuning knobs configurable
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 51s

The thresholds Part H says to tune on the real inference hardware were
hardcoded, so every adjustment would have been a PR. They now live in a
`tuning:` section of a YAML config file, which also replaces flat
environment variables as the home for structure (#32).

- The file (NISSE_CONFIG, default ./nisse-dev.yaml) carries structure
  and holds no secrets: credentials are ${VAR} references resolved from
  the environment, and a reference to an unset variable is an error
- The environment still wins: every existing NISSE_* variable overrides
  its key, and the file is optional, so .env and CI work unchanged
- Validation stays fail-fast and reports every problem at once; unknown
  keys are errors, so a misspelt knob fails at startup instead of
  silently doing nothing
- The config file joins the state store and .env in the check that
  refuses to start when they sit inside the workspace clone
- Tuning covers the executor's turn and failure limits, the consult
  budget, the delegation caps, the command timeout and the tool output
  limit, which Workspace now takes as a setting; the defaults are the
  values the agent already ran with
- llm.max_output_tokens reaches both LLM clients from the config

docs/Tuning-Guide.md explains the knobs for someone learning local
model tuning: how weights, quantization and the KV cache spend VRAM,
what each knob does and what goes wrong at either extreme, and an order
to tune in. nisse-dev.example.yaml documents every key and is checked
by a test so it cannot drift.

Closes #32

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
dongarandClaude Opus 5.5 committed 2026-09-23 23:14:07 +00:00
1 parent 4a9354ae3b
commit df449c7e34
12 files changed
+994 -67

No files matched your search

+177
View File
@@ -130,3 +130,180 @@ def test_sibling_paths_are_fine(tmp_path: Path) -> None:
}
)
assert config.workspace.name == "project"
# --- the YAML file (#32) ---------------------------------------------------
FILE = """
state_db: ~/demo/state.db
workspace: ~/demo/work
project: greet
llm:
base_url: https://litellm.test
token: ${TEST_LLM_TOKEN}
roles:
executor: local-fast
planner: claude-opus-5
delegate: claude-opus-5
consultant: claude-haiku-4-5
matrix:
homeserver: https://matrix.test/
user_id: "@nisse-dev-testing:test"
token: ${TEST_MATRIX_TOKEN}
room_id: "!room:test"
operators: ["@don:test", " "]
gitea:
url: https://gitea.test
token: ${TEST_GITEA_TOKEN}
repo: don/demo
operators: [don]
"""
SECRETS = {
"TEST_LLM_TOKEN": "sk-x",
"TEST_MATRIX_TOKEN": "syt_x",
"TEST_GITEA_TOKEN": "gt",
}
def write(tmp_path: Path, text: str) -> Path:
path = tmp_path / "nisse-dev.yaml"
path.write_text(text)
return path
def test_a_file_configures_everything_with_secrets_by_reference(
tmp_path: Path,
) -> None:
config = AgentConfig.from_sources(SECRETS, write(tmp_path, FILE))
assert config.llm_token == "sk-x"
assert config.gitea_token == "gt"
assert config.roles.executor == "local-fast"
assert config.operators == {"@don:test"} # a blank entry admits no one
assert config.matrix.homeserver == "https://matrix.test"
assert config.tuning.executor.max_turns == 24 # absent section: defaults
def test_the_environment_wins_over_the_file(tmp_path: Path) -> None:
config = AgentConfig.from_sources(
{**SECRETS, "NISSE_ROLE_EXECUTOR": "claude-haiku-4-5"}, write(tmp_path, FILE)
)
assert config.roles.executor == "claude-haiku-4-5"
def test_an_overridden_key_needs_no_resolvable_reference(tmp_path: Path) -> None:
env = {k: v for k, v in SECRETS.items() if k != "TEST_LLM_TOKEN"}
config = AgentConfig.from_sources(
{**env, "NISSE_LLM_TOKEN": "sk-env"}, write(tmp_path, FILE)
)
assert config.llm_token == "sk-env"
def test_a_reference_to_an_unset_variable_is_an_error(tmp_path: Path) -> None:
env = {k: v for k, v in SECRETS.items() if k != "TEST_GITEA_TOKEN"}
with pytest.raises(ConfigError, match=r"gitea.token: references \$\{TEST_GITEA"):
AgentConfig.from_sources(env, write(tmp_path, FILE))
def test_tuning_is_read_from_the_file(tmp_path: Path) -> None:
text = (
FILE
+ """
tuning:
executor: {max_turns: 40, failure_limit: 3}
expert: {consult_budget: 5}
delegation: {max_turns: 20, max_tokens: 100000, wall_clock_s: 600}
tools: {command_timeout_s: 300, max_output_chars: 2000}
"""
)
tuning = AgentConfig.from_sources(SECRETS, write(tmp_path, text)).tuning
assert (tuning.executor.max_turns, tuning.executor.failure_limit) == (40, 3)
assert tuning.expert.consult_budget == 5
assert tuning.delegation.max_tokens == 100_000
assert tuning.delegation.wall_clock_s == 600.0
assert tuning.tools.max_output_chars == 2000
# A section that is only partly given keeps defaults for the rest.
partial = FILE + "\ntuning:\n executor: {max_turns: 40}\n"
tuning = AgentConfig.from_sources(SECRETS, write(tmp_path, partial)).tuning
assert tuning.executor.failure_limit == 2
def test_every_problem_in_a_file_is_reported_at_once(tmp_path: Path) -> None:
# A misspelt knob that was silently ignored would look exactly like a knob
# with no effect, so unknown keys are errors.
text = (
FILE.replace("project: greet\n", "")
+ """
llm_extra: 1
tuning:
executor: {max_turn: 40, failure_limit: 0}
expert: {consult_budget: true}
bogus: {x: 1}
"""
)
with pytest.raises(ConfigError) as excinfo:
AgentConfig.from_sources(SECRETS, write(tmp_path, text))
message = str(excinfo.value)
for expected in (
"project (NISSE_PROJECT)",
"llm_extra: unknown setting",
"tuning.executor.max_turn: unknown setting",
"tuning.executor.failure_limit: must be a positive number",
"tuning.expert.consult_budget: must be a positive number",
"tuning.bogus: unknown section",
):
assert expected in message
def test_a_whole_number_is_required_where_one_is_meant(tmp_path: Path) -> None:
text = FILE + "\ntuning:\n executor: {max_turns: 2.5}\n"
with pytest.raises(ConfigError, match="tuning.executor.max_turns"):
AgentConfig.from_sources(SECRETS, write(tmp_path, text))
def test_max_output_tokens_comes_from_the_file_or_the_environment(
tmp_path: Path,
) -> None:
text = FILE.replace(
" base_url: https://litellm.test\n",
" base_url: https://litellm.test\n max_output_tokens: 8192\n",
)
path = write(tmp_path, text)
assert AgentConfig.from_sources(SECRETS, path).max_output_tokens == 8192
env = {**SECRETS, "NISSE_MAX_OUTPUT_TOKENS": "4096"}
assert AgentConfig.from_sources(env, path).max_output_tokens == 4096
def test_load_finds_the_named_file_and_refuses_a_missing_one(tmp_path: Path) -> None:
path = write(tmp_path, FILE)
assert AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(path)}).project == "greet"
with pytest.raises(ConfigError, match="missing"):
AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(tmp_path / "absent.yaml")})
def test_load_without_a_file_uses_the_environment(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
monkeypatch.chdir(tmp_path) # no ./nisse-dev.yaml here
assert AgentConfig.load(COMPLETE).config_file is None
def test_a_config_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
work = tmp_path / "project"
work.mkdir()
text = FILE.replace("workspace: ~/demo/work", f"workspace: {work}")
path = work / "nisse-dev.yaml"
path.write_text(text)
with pytest.raises(ConfigError, match="NISSE_CONFIG"):
AgentConfig.from_sources(SECRETS, path)
def test_the_example_file_is_valid(tmp_path: Path) -> None:
# The example documents every knob; it must not drift out of loading.
example = Path(__file__).resolve().parents[2] / "nisse-dev.example.yaml"
env = {
"NISSE_LLM_TOKEN": "sk-x",
"NISSE_MATRIX_TOKEN": "syt_x",
"NISSE_GITEA_TOKEN": "gt",
}
config = AgentConfig.from_sources(env, example)
assert config.tuning.executor.max_turns == 24