"""Export the action classifier's judgements from a state store (L4, M3.2). uv run python scripts/classifier-evals.py STATE_DB [...] > cases.jsonl uv run python scripts/classifier-evals.py STATE_DB --overridden One JSON line per distinct judgement: exactly what the classifier was shown, its verdict and reason, the model, what happened (ran, asked, allowed, refused), and whether the operator overrode it. Overridden cases are the ones worth adding to tests/integration/classifier_cases.yaml, with the verdict the operator's choice implies. The store is opened read-only, so a live agent's database is safe to read. """ import argparse import json import sqlite3 import sys from dataclasses import asdict from pathlib import Path from nisse_core.log.canonical import StoredEvent, from_payload from nisse_dev.evals import classifier_cases def events(path: Path) -> list[StoredEvent]: conn = sqlite3.connect(f"file:{path}?mode=ro", uri=True) rows = conn.execute( "SELECT id, ts, type, payload, work_item, actor FROM canonical_events " "WHERE type = 'action_verdict' ORDER BY id" ).fetchall() conn.close() return [ StoredEvent( id=row[0], ts=row[1], event=from_payload(row[2], json.loads(row[3])), work_item=row[4], actor=row[5], ) for row in rows ] def main() -> int: parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) parser.add_argument("stores", nargs="+", type=Path) parser.add_argument( "--overridden", action="store_true", help="only cases the operator overrode" ) args = parser.parse_args() for store in args.stores: for case in classifier_cases(events(store)): if args.overridden and not case.overridden: continue print(json.dumps({"store": str(store), **asdict(case)})) return 0 if __name__ == "__main__": sys.exit(main())