#!/usr/bin/env bash
# Evaluate a model for the driver role, from a declared LiteLLM alias to a
# benchmark report next to every model measured before it.
#
#   scripts/evaluate_model.sh ALIAS "how it is served" [RUNS] [--config PATH]
#
# 1. The context probe: does the server's window match what LiteLLM declares?
# 2. The capability matrix: does it hold the tool contract, and is its
#    supports_tool_choice declaration true?
# 3. The benchmark: canned milestones, RUNS times each (default 5).
# Stops at the first step that fails; there is no point benchmarking a model
# that can't hold the tool contract.
set -euo pipefail
cd "$(dirname "$0")/.."

alias="${1:?usage: evaluate_model.sh ALIAS NOTE [RUNS] [--config PATH]}"
note="${2:?give a note on how the model is served, e.g. \"vLLM 0.29, AWQ\"}"
runs="${3:-5}"
shift $(( $# < 3 ? $# : 3 ))
config=("$@")

echo "== 1/3 context probe"
uv run python scripts/probe_context.py "$alias" "${config[@]}"

echo "== 2/3 capability matrix"
uv run pytest -m integration -q -rs tests/integration/test_model_capability.py \
    --capability-models="$alias"

echo "== 3/3 benchmark ($runs runs per task)"
uv run python scripts/benchmark.py run --models "$alias" --runs "$runs" \
    --note "$note" "${config[@]}"