Move configuration to a YAML file, and make the tuning knobs configurable
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 51s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 51s
The thresholds Part H says to tune on the real inference hardware were hardcoded, so every adjustment would have been a PR. They now live in a `tuning:` section of a YAML config file, which also replaces flat environment variables as the home for structure (#32). - The file (NISSE_CONFIG, default ./nisse-dev.yaml) carries structure and holds no secrets: credentials are ${VAR} references resolved from the environment, and a reference to an unset variable is an error - The environment still wins: every existing NISSE_* variable overrides its key, and the file is optional, so .env and CI work unchanged - Validation stays fail-fast and reports every problem at once; unknown keys are errors, so a misspelt knob fails at startup instead of silently doing nothing - The config file joins the state store and .env in the check that refuses to start when they sit inside the workspace clone - Tuning covers the executor's turn and failure limits, the consult budget, the delegation caps, the command timeout and the tool output limit, which Workspace now takes as a setting; the defaults are the values the agent already ran with - llm.max_output_tokens reaches both LLM clients from the config docs/Tuning-Guide.md explains the knobs for someone learning local model tuning: how weights, quantization and the KV cache spend VRAM, what each knob does and what goes wrong at either extreme, and an order to tune in. nisse-dev.example.yaml documents every key and is checked by a test so it cannot drift. Closes #32 Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
1 parent
4a9354ae3b
commit
df449c7e34
12 files changed
+994
-67
No files matched your search
@@ -130,3 +130,180 @@ def test_sibling_paths_are_fine(tmp_path: Path) -> None:
|
||||
}
|
||||
)
|
||||
assert config.workspace.name == "project"
|
||||
|
||||
|
||||
# --- the YAML file (#32) ---------------------------------------------------
|
||||
|
||||
FILE = """
|
||||
state_db: ~/demo/state.db
|
||||
workspace: ~/demo/work
|
||||
project: greet
|
||||
llm:
|
||||
base_url: https://litellm.test
|
||||
token: ${TEST_LLM_TOKEN}
|
||||
roles:
|
||||
executor: local-fast
|
||||
planner: claude-opus-5
|
||||
delegate: claude-opus-5
|
||||
consultant: claude-haiku-4-5
|
||||
matrix:
|
||||
homeserver: https://matrix.test/
|
||||
user_id: "@nisse-dev-testing:test"
|
||||
token: ${TEST_MATRIX_TOKEN}
|
||||
room_id: "!room:test"
|
||||
operators: ["@don:test", " "]
|
||||
gitea:
|
||||
url: https://gitea.test
|
||||
token: ${TEST_GITEA_TOKEN}
|
||||
repo: don/demo
|
||||
operators: [don]
|
||||
"""
|
||||
SECRETS = {
|
||||
"TEST_LLM_TOKEN": "sk-x",
|
||||
"TEST_MATRIX_TOKEN": "syt_x",
|
||||
"TEST_GITEA_TOKEN": "gt",
|
||||
}
|
||||
|
||||
|
||||
def write(tmp_path: Path, text: str) -> Path:
|
||||
path = tmp_path / "nisse-dev.yaml"
|
||||
path.write_text(text)
|
||||
return path
|
||||
|
||||
|
||||
def test_a_file_configures_everything_with_secrets_by_reference(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
config = AgentConfig.from_sources(SECRETS, write(tmp_path, FILE))
|
||||
assert config.llm_token == "sk-x"
|
||||
assert config.gitea_token == "gt"
|
||||
assert config.roles.executor == "local-fast"
|
||||
assert config.operators == {"@don:test"} # a blank entry admits no one
|
||||
assert config.matrix.homeserver == "https://matrix.test"
|
||||
assert config.tuning.executor.max_turns == 24 # absent section: defaults
|
||||
|
||||
|
||||
def test_the_environment_wins_over_the_file(tmp_path: Path) -> None:
|
||||
config = AgentConfig.from_sources(
|
||||
{**SECRETS, "NISSE_ROLE_EXECUTOR": "claude-haiku-4-5"}, write(tmp_path, FILE)
|
||||
)
|
||||
assert config.roles.executor == "claude-haiku-4-5"
|
||||
|
||||
|
||||
def test_an_overridden_key_needs_no_resolvable_reference(tmp_path: Path) -> None:
|
||||
env = {k: v for k, v in SECRETS.items() if k != "TEST_LLM_TOKEN"}
|
||||
config = AgentConfig.from_sources(
|
||||
{**env, "NISSE_LLM_TOKEN": "sk-env"}, write(tmp_path, FILE)
|
||||
)
|
||||
assert config.llm_token == "sk-env"
|
||||
|
||||
|
||||
def test_a_reference_to_an_unset_variable_is_an_error(tmp_path: Path) -> None:
|
||||
env = {k: v for k, v in SECRETS.items() if k != "TEST_GITEA_TOKEN"}
|
||||
with pytest.raises(ConfigError, match=r"gitea.token: references \$\{TEST_GITEA"):
|
||||
AgentConfig.from_sources(env, write(tmp_path, FILE))
|
||||
|
||||
|
||||
def test_tuning_is_read_from_the_file(tmp_path: Path) -> None:
|
||||
text = (
|
||||
FILE
|
||||
+ """
|
||||
tuning:
|
||||
executor: {max_turns: 40, failure_limit: 3}
|
||||
expert: {consult_budget: 5}
|
||||
delegation: {max_turns: 20, max_tokens: 100000, wall_clock_s: 600}
|
||||
tools: {command_timeout_s: 300, max_output_chars: 2000}
|
||||
"""
|
||||
)
|
||||
tuning = AgentConfig.from_sources(SECRETS, write(tmp_path, text)).tuning
|
||||
assert (tuning.executor.max_turns, tuning.executor.failure_limit) == (40, 3)
|
||||
assert tuning.expert.consult_budget == 5
|
||||
assert tuning.delegation.max_tokens == 100_000
|
||||
assert tuning.delegation.wall_clock_s == 600.0
|
||||
assert tuning.tools.max_output_chars == 2000
|
||||
# A section that is only partly given keeps defaults for the rest.
|
||||
partial = FILE + "\ntuning:\n executor: {max_turns: 40}\n"
|
||||
tuning = AgentConfig.from_sources(SECRETS, write(tmp_path, partial)).tuning
|
||||
assert tuning.executor.failure_limit == 2
|
||||
|
||||
|
||||
def test_every_problem_in_a_file_is_reported_at_once(tmp_path: Path) -> None:
|
||||
# A misspelt knob that was silently ignored would look exactly like a knob
|
||||
# with no effect, so unknown keys are errors.
|
||||
text = (
|
||||
FILE.replace("project: greet\n", "")
|
||||
+ """
|
||||
llm_extra: 1
|
||||
tuning:
|
||||
executor: {max_turn: 40, failure_limit: 0}
|
||||
expert: {consult_budget: true}
|
||||
bogus: {x: 1}
|
||||
"""
|
||||
)
|
||||
with pytest.raises(ConfigError) as excinfo:
|
||||
AgentConfig.from_sources(SECRETS, write(tmp_path, text))
|
||||
message = str(excinfo.value)
|
||||
for expected in (
|
||||
"project (NISSE_PROJECT)",
|
||||
"llm_extra: unknown setting",
|
||||
"tuning.executor.max_turn: unknown setting",
|
||||
"tuning.executor.failure_limit: must be a positive number",
|
||||
"tuning.expert.consult_budget: must be a positive number",
|
||||
"tuning.bogus: unknown section",
|
||||
):
|
||||
assert expected in message
|
||||
|
||||
|
||||
def test_a_whole_number_is_required_where_one_is_meant(tmp_path: Path) -> None:
|
||||
text = FILE + "\ntuning:\n executor: {max_turns: 2.5}\n"
|
||||
with pytest.raises(ConfigError, match="tuning.executor.max_turns"):
|
||||
AgentConfig.from_sources(SECRETS, write(tmp_path, text))
|
||||
|
||||
|
||||
def test_max_output_tokens_comes_from_the_file_or_the_environment(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
text = FILE.replace(
|
||||
" base_url: https://litellm.test\n",
|
||||
" base_url: https://litellm.test\n max_output_tokens: 8192\n",
|
||||
)
|
||||
path = write(tmp_path, text)
|
||||
assert AgentConfig.from_sources(SECRETS, path).max_output_tokens == 8192
|
||||
env = {**SECRETS, "NISSE_MAX_OUTPUT_TOKENS": "4096"}
|
||||
assert AgentConfig.from_sources(env, path).max_output_tokens == 4096
|
||||
|
||||
|
||||
def test_load_finds_the_named_file_and_refuses_a_missing_one(tmp_path: Path) -> None:
|
||||
path = write(tmp_path, FILE)
|
||||
assert AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(path)}).project == "greet"
|
||||
with pytest.raises(ConfigError, match="missing"):
|
||||
AgentConfig.load({**SECRETS, "NISSE_CONFIG": str(tmp_path / "absent.yaml")})
|
||||
|
||||
|
||||
def test_load_without_a_file_uses_the_environment(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.chdir(tmp_path) # no ./nisse-dev.yaml here
|
||||
assert AgentConfig.load(COMPLETE).config_file is None
|
||||
|
||||
|
||||
def test_a_config_file_inside_the_clone_is_refused(tmp_path: Path) -> None:
|
||||
work = tmp_path / "project"
|
||||
work.mkdir()
|
||||
text = FILE.replace("workspace: ~/demo/work", f"workspace: {work}")
|
||||
path = work / "nisse-dev.yaml"
|
||||
path.write_text(text)
|
||||
with pytest.raises(ConfigError, match="NISSE_CONFIG"):
|
||||
AgentConfig.from_sources(SECRETS, path)
|
||||
|
||||
|
||||
def test_the_example_file_is_valid(tmp_path: Path) -> None:
|
||||
# The example documents every knob; it must not drift out of loading.
|
||||
example = Path(__file__).resolve().parents[2] / "nisse-dev.example.yaml"
|
||||
env = {
|
||||
"NISSE_LLM_TOKEN": "sk-x",
|
||||
"NISSE_MATRIX_TOKEN": "syt_x",
|
||||
"NISSE_GITEA_TOKEN": "gt",
|
||||
}
|
||||
config = AgentConfig.from_sources(env, example)
|
||||
assert config.tuning.executor.max_turns == 24
|
||||
Reference in new issue
Block a user