Fix the run-breaking tool-result order; e2e test runs the slice (M2.1, PR D)
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 2m17s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 2m17s
The first live run of the new loop's e2e test found that a fact recorded while start_milestone was in flight landed between the tool call and its result; both API formats reject that, so every later request in the run failed. Views now keep each result right after its call. Scripted test models check the APIs' ordering rules, so this can't pass unit tests again. The e2e tests no longer depend on the sandbox's history: each run asks for a uniquely named function and works against a base branch of its own, deleted afterwards. The review step asks for a change that can't already be there. The scripted operator gives up after five fruitless answers, and each run keeps its state store for inspection. The driver's prompt says review fixes stay in the current milestone. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
1 parent
30ab5c9b9c
commit
37965aefca
11 files changed
+203
-37
No files matched your search
@@ -25,6 +25,8 @@ from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import pytest
|
||||
from e2e_support import base_branch, operator_gitea
|
||||
from e2e_support import task as make_task
|
||||
|
||||
from nisse_dev.agent import Agent, build_agent
|
||||
from nisse_dev.canonical import UserMessage
|
||||
@@ -43,10 +45,6 @@ pytestmark = [pytest.mark.integration, pytest.mark.e2e]
|
||||
|
||||
OPERATOR = "@dongar:dongar.net" # an allowlisted operator in nisse-test.yaml
|
||||
SETTLE_S = 1200 # a milestone, with real models, inside one message
|
||||
TASK = (
|
||||
"Add a function `count_vowels(text)` to the sandbox package that returns "
|
||||
"how many of a, e, i, o, u (either case) the text contains, with tests."
|
||||
)
|
||||
|
||||
|
||||
def build_host(ssh_host: SSHHost) -> BuildHost:
|
||||
@@ -70,10 +68,14 @@ def agent(
|
||||
ssh_host: SSHHost,
|
||||
tmp_path: Path,
|
||||
monkeypatch: pytest.MonkeyPatch,
|
||||
) -> Iterator[tuple[Agent, InMemoryTransport]]:
|
||||
) -> Iterator[tuple[Agent, InMemoryTransport, str]]:
|
||||
run = uuid.uuid4().hex[:8]
|
||||
stack = contextlib.ExitStack()
|
||||
operator = operator_gitea(live_config.gitea_url)
|
||||
base = stack.enter_context(base_branch(operator, live_config.gitea_repo, run))
|
||||
config = replace(
|
||||
live_config,
|
||||
base_branch=base,
|
||||
state_db=tmp_path / "state.db",
|
||||
workspace=Path(f"/tmp/nisse-e2e-{run}/sandbox"),
|
||||
build_host=build_host(ssh_host),
|
||||
@@ -84,7 +86,7 @@ def agent(
|
||||
try:
|
||||
agent = build_agent(config)
|
||||
agent.start()
|
||||
yield agent, transport
|
||||
yield agent, transport, run
|
||||
finally:
|
||||
if agent is not None:
|
||||
agent.stop()
|
||||
@@ -97,6 +99,7 @@ def agent(
|
||||
state_dir=tmp_path / "ssh",
|
||||
)
|
||||
host.execute(f"rm -rf /tmp/nisse-e2e-{run}")
|
||||
stack.close() # the run's base branch
|
||||
|
||||
|
||||
class Operator:
|
||||
@@ -148,14 +151,16 @@ def is_artifact(path: str) -> bool:
|
||||
)
|
||||
|
||||
|
||||
def test_a_milestone_end_to_end(agent: tuple[Agent, InMemoryTransport]) -> None:
|
||||
running, transport = agent
|
||||
def test_a_milestone_end_to_end(
|
||||
agent: tuple[Agent, InMemoryTransport, str],
|
||||
) -> None:
|
||||
running, transport, run = agent
|
||||
operator = Operator(running, transport)
|
||||
gitea, repo = running.gitea, running.config.gitea_repo
|
||||
pr: dict[str, Any] = {}
|
||||
try:
|
||||
# The task first, before any project: it must be kept, not lost.
|
||||
operator.say(TASK)
|
||||
operator.say(make_task(run))
|
||||
operator.say(f"use {running.config.project}")
|
||||
state = operator.drive_to_pr()
|
||||
pr = state.data["pr"]
|
||||
|
||||
Reference in new issue
Block a user