"""`ask_expert` (architecture L9). The one case delegation does not cover: a question needing no repo access — "which of these two approaches is better", "is this API deprecated". A single stateless call, a text answer, no session state, no diff application. Two constraints from L9 shape the implementation: **Structured output is contracted at the prompt level, not the API level.** Vendors differ on JSON mode and schema enforcement, so the answer is parsed and validated against a local schema, with one retry. This runs over LiteLLM's *normalized* route, which L9 reserves for calls where fidelity is cheap. **A per-task budget**, so a confused executor cannot spend twenty consults circling. The count comes from the canonical log rather than a counter in memory: it survives a restart, and it is observable from outside the model, which is the same reasoning L9 applies to escalation triggers. Consults are recorded as ``delegation_started`` / ``delegation_completed`` with role ``expert``. That is deliberate — L9 wants a "consult/delegation eval set", and reusing the composite events gets consult evaluation for free rather than inventing a parallel record. """ import json import re from dataclasses import dataclass from nisse_core.canonical import ( DelegationCompleted, DelegationStarted, StoredEvent, UserMessage, ) from nisse_core.llm import LLMClient, LLMError, RoleMap from nisse_core.state import Transcript EXPERT_ROLE = "expert" DEFAULT_BUDGET = 3 SYSTEM_PROMPT = """You are a consultant answering one self-contained question. You have no access to the caller's repository, so answer from the question alone. If the question cannot be answered without seeing the code, say so plainly in `answer` rather than guessing. Reply with a single JSON object and nothing else: {"answer": "...", "confidence": "high" | "medium" | "low", "caveats": "..."} Keep `answer` direct and specific. Put what would change your mind, or what you are assuming, in `caveats`.""" _RETRY_NUDGE = ( "\n\nYour previous reply was not a single valid JSON object with the keys " "answer, confidence and caveats. Reply with only that object." ) _CONFIDENCES = {"high", "medium", "low"} ASK_EXPERT_TOOL: dict = { "type": "function", "function": { "name": "ask_expert", "description": ( "Ask a stronger model one self-contained question that needs no " "access to this repository — a design trade-off, or how a library " "behaves. It cannot see your files. Use sparingly; the budget is " "small." ), "parameters": { "type": "object", "properties": {"question": {"type": "string"}}, "required": ["question"], }, }, } class BudgetExhausted(RuntimeError): """Raised rather than silently answering: a caller that has burned its consult budget is circling, and should escalate or ask the operator.""" @dataclass(frozen=True) class ExpertAnswer: answer: str confidence: str caveats: str def render(self) -> str: body = f"{self.answer}\n\n(confidence: {self.confidence})" return f"{body}\n\nCaveats: {self.caveats}" if self.caveats else body class ExpertDesk: def __init__( self, client: LLMClient, roles: RoleMap, *, budget: int = DEFAULT_BUDGET, ) -> None: self._client = client self._roles = roles self._budget = budget def consults_used(self, transcript: Transcript, since_event_id: int = 0) -> int: return sum( 1 for e in transcript.read(after=since_event_id) if isinstance(e.event, DelegationStarted) and e.event.role == EXPERT_ROLE ) def ask( self, transcript: Transcript, question: str, *, since_event_id: int = 0 ) -> ExpertAnswer: """The consult is recorded in the work item's transcript, as the expert, and the budget counts that work item's consults (L9).""" used = self.consults_used(transcript, since_event_id) if used >= self._budget: raise BudgetExhausted( f"consult budget of {self._budget} is spent for this task " f"({used} used); escalate or ask the operator instead" ) model = self._roles.resolve(EXPERT_ROLE) transcript.append( DelegationStarted( brief=question, role=EXPERT_ROLE, model=model, caps={"consults": self._budget, "used": used}, ) ) answer, note = self._answer(question, model) transcript.append( DelegationCompleted( handback=answer.render() if answer else note, artifacts=[], tokens=0, duration_s=0.0, exit_status="answered" if answer else "failed", ) ) if answer is None: raise LLMError(f"ask_expert produced no usable answer: {note}") return answer def _answer(self, question: str, model: str) -> tuple[ExpertAnswer | None, str]: system = SYSTEM_PROMPT last = "no reply" for attempt in (1, 2): try: # Stateless by design (L9): the question travels as a # synthesised message that is never appended to the log, so a # consult neither depends on nor leaks repo context. It cannot # ride in the system prompt alone — the Messages API requires # at least one non-system message. produced = self._client.complete( [ StoredEvent( id=0, ts="", event=UserMessage(text=question, source="ask_expert"), ) ], tools=None, model=model, system=system, ) except LLMError as exc: return None, str(exc) text = "\n".join( getattr(e, "text", "") for e in produced if hasattr(e, "text") ).strip() parsed = parse_answer(text) if parsed is not None: return parsed, "ok" last = text[:300] or "empty reply" if attempt == 1: system = SYSTEM_PROMPT + _RETRY_NUDGE return None, f"unparseable reply: {last}" def parse_answer(text: str) -> ExpertAnswer | None: """Validated locally because the prompt-level contract is all we have. Tolerates a fenced code block, since models wrap JSON in one habitually, but not missing or malformed fields — those are a failed turn. """ candidate = text.strip() fenced = re.search(r"```(?:json)?\s*(.*?)```", candidate, re.DOTALL) if fenced: candidate = fenced.group(1).strip() start, end = candidate.find("{"), candidate.rfind("}") if start == -1 or end <= start: return None try: data = json.loads(candidate[start : end + 1]) except json.JSONDecodeError: return None if not isinstance(data, dict): return None answer = data.get("answer") confidence = data.get("confidence") if not isinstance(answer, str) or not answer.strip(): return None if not isinstance(confidence, str) or confidence.lower() not in _CONFIDENCES: return None caveats = data.get("caveats") return ExpertAnswer( answer=answer.strip(), confidence=confidence.lower(), caveats=caveats.strip() if isinstance(caveats, str) else "", )