dongarandClaude Opus 5.5
3b614bd918
Five runs per task through LiteLLM on the RTX 4090. qwen3.8-27b (thinking) passed 36 of 40. Flash-Next was run thinking, at reasoning_effort low, and with no thinking. The first Flash-Next runs used non-git workspaces, where `git log` fails, so their task hashes are older and the report marks them (old). On git workspaces (#177), both thinking and lowthink passed 32 of 40; nothink passed 12 of 40 on the old workspaces and was not rerun. qwen3.8-27b had no git-caused failures, so it was not rerun. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01J5Utyfzw7gF7UhjRDEtjTk
41 lines
64 KiB
JSON
41 lines
64 KiB
JSON
{"backing": "openai/qwen3.8-27b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "fix-bug", "task_hash": "9e9349d5ed6dae8d", "tokens_in": 17195, "tokens_out": 1262, "trimmed_requests": 0, "ts": "2026-10-23T22:19:03+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 30.5}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "fix-bug", "task_hash": "9e9349d5ed6dae8d", "tokens_in": 15947, "tokens_out": 1083, "trimmed_requests": 0, "ts": "2026-10-23T22:19:29+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 25.6}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "fix-bug", "task_hash": "9e9349d5ed6dae8d", "tokens_in": 13486, "tokens_out": 800, "trimmed_requests": 0, "ts": "2026-10-23T22:19:49+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 19.5}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "fix-bug", "task_hash": "9e9349d5ed6dae8d", "tokens_in": 25777, "tokens_out": 2346, "trimmed_requests": 0, "ts": "2026-10-23T22:20:43+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 54.2}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "...... [100%]\n6 passed in 0.01s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "fix-bug", "task_hash": "9e9349d5ed6dae8d", "tokens_in": 15542, "tokens_out": 1022, "trimmed_requests": 0, "ts": "2026-10-23T22:21:08+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 24.5}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": ".................... [100%]\n20 passed in 1.06s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "long-milestone", "task_hash": "106e68dd7bfe9dba", "tokens_in": 240954, "tokens_out": 35142, "trimmed_requests": 0, "ts": "2026-10-23T22:34:32+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 27, "wall_s": 803.0}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": ".................... [100%]\n20 passed in 0.66s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "long-milestone", "task_hash": "106e68dd7bfe9dba", "tokens_in": 113267, "tokens_out": 29566, "trimmed_requests": 0, "ts": "2026-10-23T22:45:41+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 17, "wall_s": 667.5}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": ".................... [100%]\n20 passed in 0.63s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "long-milestone", "task_hash": "106e68dd7bfe9dba", "tokens_in": 170108, "tokens_out": 14783, "trimmed_requests": 0, "ts": "2026-10-23T22:51:21+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 25, "wall_s": 339.8}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": ".................... [100%]\n20 passed in 0.62s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "long-milestone", "task_hash": "106e68dd7bfe9dba", "tokens_in": 164089, "tokens_out": 29457, "trimmed_requests": 0, "ts": "2026-10-23T23:02:32+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 23, "wall_s": 669.5}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": ".................... [100%]\n20 passed in 0.62s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "long-milestone", "task_hash": "106e68dd7bfe9dba", "tokens_in": 248035, "tokens_out": 32002, "trimmed_requests": 0, "ts": "2026-10-23T23:14:45+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 27, "wall_s": 732.3}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "........................................................................ [ 23%]\n........................................................................ [ 47%]\n........................................................................ [ 71%]\n........................................................................ [ 95%]\n............. [100%]\n301 passed in 0.25s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "long-output", "task_hash": "b90340b3dd8b39b4", "tokens_in": 55307, "tokens_out": 3038, "trimmed_requests": 0, "ts": "2026-10-23T23:15:58+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 72.7}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "........................................................................ [ 23%]\n........................................................................ [ 47%]\n........................................................................ [ 71%]\n........................................................................ [ 95%]\n............. [100%]\n301 passed in 0.26s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "long-output", "task_hash": "b90340b3dd8b39b4", "tokens_in": 35279, "tokens_out": 2612, "trimmed_requests": 0, "ts": "2026-10-23T23:17:00+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 61.4}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "........................................................................ [ 23%]\n........................................................................ [ 47%]\n........................................................................ [ 71%]\n........................................................................ [ 95%]\n............. [100%]\n301 passed in 0.25s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "stuck", "outcome_note": "2 commands failed in a row: `git diff`: cified path\n --find-object <object-id>\n look for differences that change the number of occurrences of the specified object\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\n select files by diff type\n --output <file> Output to a specific file; `cd /tmp/nisse-bench/2a432495ccc048f6b3cb397da1e292db && git diff`: cified path\n --find-object <object-id>\n look for differences t", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "long-output", "task_hash": "b90340b3dd8b39b4", "tokens_in": 24503, "tokens_out": 996, "trimmed_requests": 0, "ts": "2026-10-23T23:17:26+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 25.7}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "........................................................................ [ 23%]\n........................................................................ [ 47%]\n........................................................................ [ 71%]\n........................................................................ [ 95%]\n............. [100%]\n301 passed in 0.30s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "stuck", "outcome_note": "2 commands failed in a row: `cd /tmp/nisse-bench/5ece93b7ef0541be8f4326645a3c93fd && git status 2>&1 | head -5; git branch --show-current 2>&1; git l`: exit code: 128\nstdout:\nfatal: not a git repository (or any of the parent directories): .git\nfatal: not a git repository (or any of the parent directories): .git\nfatal: not a git repository (or any of the parent directories): .git\n\nstderr:; `cd /tmp/nisse-bench/5ece93b7ef0541be8f4326645a3c93fd && grep -n '\"R237\"' rates.py && cp rates.py /tmp/rates", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "long-output", "task_hash": "b90340b3dd8b39b4", "tokens_in": 31086, "tokens_out": 3038, "trimmed_requests": 0, "ts": "2026-10-23T23:18:38+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 70.6}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "........................................................................ [ 23%]\n........................................................................ [ 47%]\n........................................................................ [ 71%]\n........................................................................ [ 95%]\n............. [100%]\n301 passed in 0.29s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "stuck", "outcome_note": "2 commands failed in a row: `sed -i '241s/\"R237\": 0.21,/\"R237\": 0.12,/' rates.py && sed -n '239,243p' rates.py && git diff --stat`: cified path\n --find-object <object-id>\n look for differences that change the number of occurrences of the specified object\n --diff-filter [(A|C|D|M|R|T|U|X|B)...[*]]\n select files by diff type\n --output <file> Output to a specific file; `pwd; ls -la; git rev-parse --show-toplevel 2>&1; git -C /tmp/niss", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "long-output", "task_hash": "b90340b3dd8b39b4", "tokens_in": 19263, "tokens_out": 1350, "trimmed_requests": 0, "ts": "2026-10-23T23:19:10+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 32.3}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "..... [100%]\n5 passed in 0.05s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "multi-file", "task_hash": "870e2574d3aca955", "tokens_in": 25242, "tokens_out": 1990, "trimmed_requests": 0, "ts": "2026-10-23T23:19:58+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 46.7}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "..... [100%]\n5 passed in 0.03s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "multi-file", "task_hash": "870e2574d3aca955", "tokens_in": 17101, "tokens_out": 2195, "trimmed_requests": 0, "ts": "2026-10-23T23:20:49+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 50.9}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "..... [100%]\n5 passed in 0.04s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "multi-file", "task_hash": "870e2574d3aca955", "tokens_in": 13758, "tokens_out": 1705, "trimmed_requests": 0, "ts": "2026-10-23T23:21:29+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 39.8}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "..... [100%]\n5 passed in 0.03s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "multi-file", "task_hash": "870e2574d3aca955", "tokens_in": 13894, "tokens_out": 1912, "trimmed_requests": 0, "ts": "2026-10-23T23:22:13+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 43.5}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "..... [100%]\n5 passed in 0.03s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "multi-file", "task_hash": "870e2574d3aca955", "tokens_in": 13555, "tokens_out": 1333, "trimmed_requests": 0, "ts": "2026-10-23T23:22:44+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 30.9}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "....... [100%]\n7 passed in 0.26s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "refactor-rename", "task_hash": "aaa8fc0b5a74a012", "tokens_in": 109816, "tokens_out": 14227, "trimmed_requests": 0, "ts": "2026-10-23T23:28:09+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 17, "wall_s": 324.9}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "....... [100%]\n7 passed in 0.24s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "refactor-rename", "task_hash": "aaa8fc0b5a74a012", "tokens_in": 66144, "tokens_out": 5892, "trimmed_requests": 0, "ts": "2026-10-23T23:30:26+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 12, "wall_s": 135.6}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "....... [100%]\n7 passed in 0.24s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "refactor-rename", "task_hash": "aaa8fc0b5a74a012", "tokens_in": 29872, "tokens_out": 1448, "trimmed_requests": 0, "ts": "2026-10-23T23:31:01+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 34.9}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "....... [100%]\n7 passed in 0.23s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "refactor-rename", "task_hash": "aaa8fc0b5a74a012", "tokens_in": 58265, "tokens_out": 3548, "trimmed_requests": 0, "ts": "2026-10-23T23:32:24+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 12, "wall_s": 82.7}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "....... [100%]\n7 passed in 0.26s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "refactor-rename", "task_hash": "aaa8fc0b5a74a012", "tokens_in": 51177, "tokens_out": 3971, "trimmed_requests": 0, "ts": "2026-10-23T23:33:57+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 92.2}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "%]\n=================================== FAILURES ===================================\n_________________ test_the_plan_asked_for_tests_and_they_pass __________________\n\n def test_the_plan_asked_for_tests_and_they_pass() -> None:\n # At least one valid and one invalid input for each of the eight rules.\n import subprocess\n import sys\n \n run = subprocess.run(\n [sys.executable, \"-m\", \"pytest\", \"-q\", \"test_durations.py\"],\n capture_output=True,\n text=True,\n check=False,\n )\n> assert run.returncode == 0, run.stdout[-1500:]\nE AssertionError: ..F............... [100%]\nE =================================== FAILURES ===================================\nE _______________________________ test_rule2_valid _______________________________\nE \nE def test_rule2_valid():\nE assert parse_duration(\"1H30M\") == 5400\nE > assert parse_duration(\"2D1h\") == 172800\nE E AssertionError: assert 176400 == 172800\nE E + where 176400 = parse_duration('2D1h')\nE \nE test_durations.py:34: AssertionError\nE =========================== short test summary info ============================\nE FAILED test_durations.py::test_rule2_valid - AssertionError: assert 176400 ==...\nE 1 failed, 17 passed in 0.03s\nE \nE assert 1 == 0\nE + where 1 = CompletedProcess(args=['/usr/bin/python', '-m', 'pytest', '-q', 'test_durations.py'], returncode=1, stdout='..F.......... test_durations.py::test_rule2_valid - AssertionError: assert 176400 ==...\\n1 failed, 17 passed in 0.03s\\n', stderr='').returncode\n\n.bench_check/test_hidden.py:87: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_the_plan_asked_for_tests_and_they_pass\n1 failed, 48 passed in 0.30s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "stuck", "outcome_note": "2 commands failed in a row: `python -m pytest -q test_durations.py`: alueError'>\n\ntest_durations.py:10: Failed\n=========================== short test summary info ============================\nFAILED test_durations.py::test_rule2_valid - KeyError: 'H'\nFAILED test_durations.py::test_rule6_invalid - Failed: DID NOT RAISE <class '...\n2 failed, 16 passed in 0.06s\n\nstderr:; `python -m pytest -q test_durations.py`: 00 == 172800\nE + where 176400 = parse_duration('2D1h')\n\ntest_durations.py:34: As", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": false, "task": "spec-parser", "task_hash": "ab0053a41dbbcdf8", "tokens_in": 67972, "tokens_out": 26798, "trimmed_requests": 0, "ts": "2026-10-23T23:44:01+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 603.8}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "................................................F [100%]\n=================================== FAILURES ===================================\n_________________ test_the_plan_asked_for_tests_and_they_pass __________________\n\n def test_the_plan_asked_for_tests_and_they_pass() -> None:\n # At least one valid and one invalid input for each of the eight rules.\n import subprocess\n import sys\n \n run = subprocess.run(\n [sys.executable, \"-m\", \"pytest\", \"-q\", \"test_durations.py\"],\n capture_output=True,\n text=True,\n check=False,\n )\n assert run.returncode == 0, run.stdout[-1500:]\n summary = run.stdout.strip().splitlines()[-1]\n> assert int(summary.split()[0]) >= 16, summary\nE AssertionError: 10 passed in 0.02s\nE assert 10 >= 16\nE + where 10 = int('10')\n\n.bench_check/test_hidden.py:89: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_the_plan_asked_for_tests_and_they_pass\n1 failed, 48 passed in 0.30s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": false, "task": "spec-parser", "task_hash": "ab0053a41dbbcdf8", "tokens_in": 45446, "tokens_out": 24240, "trimmed_requests": 0, "ts": "2026-10-23T23:53:06+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 543.7}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "................................................. [100%]\n49 passed in 0.32s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "spec-parser", "task_hash": "ab0053a41dbbcdf8", "tokens_in": 42076, "tokens_out": 21588, "trimmed_requests": 0, "ts": "2026-10-24T00:01:12+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 8, "wall_s": 485.9}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "t_durations.py::test_unit_form_valid[45s-45] - ValueError: invalid ...\nE FAILED test_durations.py::test_case_valid - ValueError: invalid duration: '1d...\nE FAILED test_durations.py::test_decimals_valid[1.5h-5400] - ValueError: invali...\nE FAILED test_durations.py::test_decimals_valid[1.5m-90] - ValueError: invalid ...\nE FAILED test_durations.py::test_decimals_valid[0.5m-30] - ValueError: invalid ...\nE FAILED test_durations.py::test_zero_valid - ValueError: invalid duration: '0s'\nE 8 failed, 48 passed in 0.10s\nE \nE assert 1 == 0\nE + where 1 = CompletedProcess(args=['/usr/bin/python', '-m', 'pytest', '-q', 'test_durations.py'], returncode=1, stdout='F.FF....F.... test_durations.py::test_zero_valid - ValueError: invalid duration: \\'0s\\'\\n8 failed, 48 passed in 0.10s\\n', stderr='').returncode\n\n.bench_check/test_hidden.py:87: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_valid[2h-7200] - ValueError: invalid...\nFAILED .bench_check/test_hidden.py::test_valid[1d2h3m4s-93784] - ValueError: ...\nFAILED .bench_check/test_hidden.py::test_valid[45s-45] - ValueError: invalid ...\nFAILED .bench_check/test_hidden.py::test_valid[1d-86400] - ValueError: invali...\nFAILED .bench_check/test_hidden.py::test_valid[2D-172800] - ValueError: inval...\nFAILED .bench_check/test_hidden.py::test_valid[1d 2h 3m 4s-93784] - ValueErro...\nFAILED .bench_check/test_hidden.py::test_valid[1.5h-5400] - ValueError: inval...\nFAILED .bench_check/test_hidden.py::test_valid[1.5m-90] - ValueError: invalid...\nFAILED .bench_check/test_hidden.py::test_valid[0.25h-900] - ValueError: inval...\nFAILED .bench_check/test_hidden.py::test_valid[0s-0] - ValueError: invalid du...\nFAILED .bench_check/test_hidden.py::test_the_result_is_an_int - ValueError: i...\nFAILED .bench_check/test_hidden.py::test_the_plan_asked_for_tests_and_they_pass\n12 failed, 37 passed in 0.37s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "stuck", "outcome_note": "2 commands failed in a row: `python -m pytest -q test_durations.py`: .py:100: ValueError\n=========================== short test summary info ============================\nFAILED test_durations.py::test_order_valid - ValueError: invalid duration: '2...\nFAILED test_durations.py::test_spaces_valid - ValueError: invalid duration: '...\n2 failed, 54 passed in 0.08s\n\nstderr:; `python -m pytest -q test_durations.py`: ValueError: invali...\nFAILED test_durations.py::test_decimals_valid[1.5m-90] - ValueErro", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": false, "task": "spec-parser", "task_hash": "ab0053a41dbbcdf8", "tokens_in": 46209, "tokens_out": 17440, "trimmed_requests": 0, "ts": "2026-10-24T00:07:44+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 391.1}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "%]\n=================================== FAILURES ===================================\n_________________ test_the_plan_asked_for_tests_and_they_pass __________________\n\n def test_the_plan_asked_for_tests_and_they_pass() -> None:\n # At least one valid and one invalid input for each of the eight rules.\n import subprocess\n import sys\n \n run = subprocess.run(\n [sys.executable, \"-m\", \"pytest\", \"-q\", \"test_durations.py\"],\n capture_output=True,\n text=True,\n check=False,\n )\n> assert run.returncode == 0, run.stdout[-1500:]\nE AssertionError: ....F............ [100%]\nE =================================== FAILURES ===================================\nE ____________________________ test_rule3_order_valid ____________________________\nE \nE def test_rule3_order_valid():\nE assert parse_duration(\"1d\") == 86400\nE > assert parse_duration(\"1d2h\") == 90000\nE E AssertionError: assert 93600 == 90000\nE E + where 93600 = parse_duration('1d2h')\nE \nE test_durations.py:43: AssertionError\nE =========================== short test summary info ============================\nE FAILED test_durations.py::test_rule3_order_valid - AssertionError: assert 936...\nE 1 failed, 16 passed in 0.03s\nE \nE assert 1 == 0\nE + where 1 = CompletedProcess(args=['/usr/bin/python', '-m', 'pytest', '-q', 'test_durations.py'], returncode=1, stdout='....F........ test_durations.py::test_rule3_order_valid - AssertionError: assert 936...\\n1 failed, 16 passed in 0.03s\\n', stderr='').returncode\n\n.bench_check/test_hidden.py:87: AssertionError\n=========================== short test summary info ============================\nFAILED .bench_check/test_hidden.py::test_the_plan_asked_for_tests_and_they_pass\n1 failed, 48 passed in 0.27s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 2, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "stuck", "outcome_note": "2 commands failed in a row: `python -m pytest -q test_durations.py`: IndexError: no such group\nFAILED test_durations.py::test_rule6_decimals_invalid - IndexError: no such g...\nFAILED test_durations.py::test_rule8_zero_valid - IndexError: no such group\nFAILED test_durations.py::test_rule8_errors_invalid - AssertionError: Regex p...\n9 failed, 8 passed in 0.08s\n\nstderr:; `cd /tmp/nisse-bench/30f587e1da734aa8bc880bc887d1fdf9 && python - <<'EOF'\nimport re\np = 'durations.py'\ns = open(p).read()`: 3600 ", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": false, "task": "spec-parser", "task_hash": "ab0053a41dbbcdf8", "tokens_in": 31297, "tokens_out": 18856, "trimmed_requests": 0, "ts": "2026-10-24T00:14:50+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 425.1}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": ".......... [100%]\n10 passed in 0.03s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "trace-bug", "task_hash": "ae05a8edb160d159", "tokens_in": 24569, "tokens_out": 5681, "trimmed_requests": 0, "ts": "2026-10-24T00:16:58+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 128.0}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": ".......... [100%]\n10 passed in 0.03s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 3, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "stuck", "outcome_note": "2 commands failed in a row: `git status --short && git diff`: exit code: 128\nstdout:\n\nstderr:\nfatal: not a git repository (or any of the parent directories): .git; `pwd && ls -la && git -C /tmp/nisse-bench/28292b6addd54c719fd768b670f42ea1 rev-parse --show-toplevel 2>&1`: root 4096 Oct 24 00:17 .\ndrwxr-xr-x 3 root root 4096 Oct 24 00:16 ..\ndrwxr-xr-x 3 root root 4096 Oct 24 00:17 .pytest_cache\ndrwxr-xr-x 3 root root 4096 Oct 24 00:17 invoicing\ndrwxr-xr-x 3 root root 4096 Oct 24 00:17 tests\nfatal", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "trace-bug", "task_hash": "ae05a8edb160d159", "tokens_in": 97180, "tokens_out": 10964, "trimmed_requests": 0, "ts": "2026-10-24T00:21:09+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 15, "wall_s": 250.5}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": ".......... [100%]\n10 passed in 0.03s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "trace-bug", "task_hash": "ae05a8edb160d159", "tokens_in": 55827, "tokens_out": 4580, "trimmed_requests": 0, "ts": "2026-10-24T00:22:54+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 11, "wall_s": 105.2}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": ".......... [100%]\n10 passed in 0.04s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "trace-bug", "task_hash": "ae05a8edb160d159", "tokens_in": 38912, "tokens_out": 5270, "trimmed_requests": 0, "ts": "2026-10-24T00:24:55+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 9, "wall_s": 119.7}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": ".......... [100%]\n10 passed in 0.03s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 1, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "trace-bug", "task_hash": "ae05a8edb160d159", "tokens_in": 48225, "tokens_out": 6534, "trimmed_requests": 0, "ts": "2026-10-24T00:27:23+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 10, "wall_s": 148.0}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "write-function", "task_hash": "c0e42f85934119cc", "tokens_in": 15550, "tokens_out": 1301, "trimmed_requests": 0, "ts": "2026-10-24T00:27:54+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 30.9}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "write-function", "task_hash": "c0e42f85934119cc", "tokens_in": 14504, "tokens_out": 1146, "trimmed_requests": 0, "ts": "2026-10-24T00:28:21+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 6, "wall_s": 26.8}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "........ [100%]\n8 passed in 0.02s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "write-function", "task_hash": "c0e42f85934119cc", "tokens_in": 9698, "tokens_out": 951, "trimmed_requests": 0, "ts": "2026-10-24T00:28:44+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 4, "wall_s": 22.1}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "write-function", "task_hash": "c0e42f85934119cc", "tokens_in": 12334, "tokens_out": 867, "trimmed_requests": 0, "ts": "2026-10-24T00:29:04+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 20.5}
|
|
{"backing": "openai/qwen3.8-27b", "check_output": "........ [100%]\n8 passed in 0.01s\n", "commit": "3caf4aa", "cost": 0.0, "failed_commands": 0, "fallbacks": [], "invented_tools": 0, "model": "qwen3.8-27b", "model_info": {"max_input_tokens": 131072, "max_output_tokens": 32768, "supports_function_calling": true, "supports_tool_choice": true}, "note": "qwen3.8-27b thinking, llama.cpp b11783 Q4_K_XL", "outcome": "completed", "outcome_note": "", "prose_turns": 0, "provider": "openai", "rate_limit_wait_s": 0.0, "success": true, "task": "write-function", "task_hash": "c0e42f85934119cc", "tokens_in": 12363, "tokens_out": 1345, "trimmed_requests": 0, "ts": "2026-10-24T00:29:36+00:00", "tuning": {"build_vm": {"retry_s": 300.0, "settle_s": 180.0}, "context": {"compact_at": 0.6, "compact_keep_turns": 3, "compact_to": 0.25, "keep_recent_turns": 6, "safety_margin": 0.1}, "delegation": {"max_tokens": 200000, "max_turns": 30, "wall_clock_s": 900.0}, "driver": {"failure_limit": 2, "max_turns": 40}, "expert": {"consult_budget": 3}, "loop": {"bug_grace_s": 600.0, "general_idle_s": 86400.0}, "memory": {"max_turns": 10}, "preflight": {"command_timeout_s": 900, "max_turns": 20}, "tools": {"command_timeout_s": 120, "max_output_chars": 4000}}, "turns": 5, "wall_s": 31.1}
|