From 5e1086c501257c43819737e57418e1d3e7648eae Mon Sep 17 00:00:00 2001 From: Don Garrett Date: Thu, 8 Oct 2026 16:37:52 +0000 Subject: [PATCH] Expert and integration roles on claude-haiku-5-5 claude-haiku-4-5 is being replaced by claude-haiku-5-5 in LiteLLM: the integration tests' roles and the example config's expert move to it. Its profile keeps the predecessor's benchmark figures, marked as such until it's measured. The plan's models-by-role line also gains the local classifier (M3.2). Results measured on 4.5 stay as they were. Co-Authored-By: Claude Opus 5.5 (1M context) Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE --- docs/Development-Plan.md | 4 ++-- nisse-dev.example.yaml | 7 ++++--- tests/integration/e2e_support.py | 2 +- tests/integration/nisse-test.yaml | 6 +++--- 4 files changed, 10 insertions(+), 9 deletions(-) diff --git a/docs/Development-Plan.md b/docs/Development-Plan.md index 4cdcf8b..334c07a 100644 --- a/docs/Development-Plan.md +++ b/docs/Development-Plan.md @@ -52,8 +52,8 @@ Only `nisse-dev` has real code; `nisse-core` and `nisse-ops` are placeholders. - **Local inference:** llama-swap on an RTX 4090 host (320 W cap). The benchmark (PR #55) chose `qwen3.8-27b` for local work. - **Models by role today:** driver and helper `claude-opus-5-5`, expert - `claude-haiku-4-5`. `qwen3.8-27b` has had no role since the executor's went - (M2.8); the driver benchmark measures where it fits. + `claude-haiku-5-5`, classifier `qwen3.8-27b` (M3.2); the driver benchmark + measures where the local model fits beyond that. - **Bring-up** runs on this development VM (`scripts/bringup.sh`), against test resources: a test repository, the test Matrix room, a budget-capped LiteLLM key; configuration in `/root/nisse-e2e/`. diff --git a/nisse-dev.example.yaml b/nisse-dev.example.yaml index 396ad34..0574027 100644 --- a/nisse-dev.example.yaml +++ b/nisse-dev.example.yaml @@ -48,7 +48,7 @@ roles: # L9: role -> LiteLLM model alias # Each work item pins the mapping when it opens. driver: claude-opus-5-5 # drives the loop: talks, restates, works helper: claude-opus-5-5 # takes over what the driver got stuck on - expert: claude-haiku-4-5 # one question at a time, no tools: cheap is enough + expert: claude-haiku-5-5 # one question at a time, no tools: cheap is enough # Reads go-aheads and judges actions the fast path doesn't know (L4); unset, # the expert's. It never sees the transcript, so the driver's model is fine. classifier: qwen3.8-27b @@ -61,10 +61,11 @@ profiles: tools: reliable (37/40 on the executor benchmark, PR #55) strengths: careful multi-step edits that follow a clear brief weaknesses: speed (minutes per hard task) and open-ended design - claude-haiku-4-5: + claude-haiku-5-5: tools: reliable strengths: quick, well-scoped changes and short answers - weaknesses: long multi-file work (15/20 on the harder benchmark tasks) + # Measured on its predecessor, claude-haiku-4-5; not benchmarked yet. + weaknesses: long multi-file work (15/20 on the harder benchmark tasks, as 4.5) claude-opus-5-5: tools: reliable strengths: design, debugging, and work other models got stuck on diff --git a/tests/integration/e2e_support.py b/tests/integration/e2e_support.py index 5200abd..9acd07c 100644 --- a/tests/integration/e2e_support.py +++ b/tests/integration/e2e_support.py @@ -16,7 +16,7 @@ from nisse_dev.gitea import GiteaClient, GiteaError from nisse_dev.llm import RoleMap # The end-to-end tests run every role on a local model, which costs nothing -# per call; NISSE_E2E_MODEL overrides it (e.g. claude-haiku-4-5 for a quick, +# per call; NISSE_E2E_MODEL overrides it (e.g. claude-haiku-5-5 for a quick, # paid run). The no-thinking alias (LiteLLM: chat_template_kwargs # enable_thinking false): with thinking, some answers ran to 6-8k tokens, # minutes each, and stretched a test from 3.5 to 20 minutes. diff --git a/tests/integration/nisse-test.yaml b/tests/integration/nisse-test.yaml index 46ab6ee..67adf7d 100644 --- a/tests/integration/nisse-test.yaml +++ b/tests/integration/nisse-test.yaml @@ -36,9 +36,9 @@ llm: # not local-model competence (measured separately by test_model_capability), # so every role uses a model that holds the tool-call contract. roles: - driver: claude-haiku-4-5 - helper: claude-haiku-4-5 - expert: claude-haiku-4-5 + driver: claude-haiku-5-5 + helper: claude-haiku-5-5 + expert: claude-haiku-5-5 matrix: homeserver: https://matrix.dongar.net