Threads, commands and labels on the new loop (M2.3)
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 3m45s
CI / checks (push) Skipped
CI / checks (pull_request) Successful in 3m45s
- Commands that name their item ("go #14", "abandon #14", "stop #14")
apply to it wherever they're said; go, approve and abandon alone at top
level are asked about (L5). stop is never taken as an answer.
- A plain-language reply to a restatement is read by GoAheadReader, one
model call that agrees only on a clear yes; questions never reach it. The
agent says "taking that as a go-ahead on rev-N" first, bound to the
revision on screen at arrival (L3, L5, D1).
- Labels steer (D1): `bug` goes ahead after tuning.loop.bug_grace_s
(default 600 s) unless the operator replies; `discuss` isn't offered
restate until the operator has spoken; `bug`, `discuss` and `design` add
to the driver's instructions. A restatement asks no questions.
Tests: label and command unit tests, the property walk with labelled
issues, plain yeses, named gos and time passing (grace_only_unanswered,
discuss_talks_first), and e2e tests for a bug proceeding after its grace
period and a plain-language go-ahead.
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01PPQ14L3LikyefCMFBqEbKE
This commit is contained in:
1 parent
a504526da2
commit
4d9c1d7529
13 files changed
+793
-35
No files matched your search
@@ -9,6 +9,7 @@ the item's Matrix thread through the in-memory transport. Run with
|
||||
scripts/e2e.sh after each milestone.
|
||||
"""
|
||||
|
||||
import base64
|
||||
import contextlib
|
||||
import os
|
||||
import time
|
||||
@@ -22,7 +23,7 @@ from typing import Any
|
||||
import pytest
|
||||
from e2e_support import base_branch, dependent_task, operator_gitea, task
|
||||
|
||||
from nisse_dev.config import AgentConfig
|
||||
from nisse_dev.config import AgentConfig, LoopTuning
|
||||
from nisse_dev.execution import SSHHost
|
||||
from nisse_dev.gitea import GiteaClient, GiteaError
|
||||
from nisse_dev.loop.build import RunningLoop, build_loop_agent
|
||||
@@ -42,6 +43,7 @@ pytestmark = [pytest.mark.integration, pytest.mark.e2e]
|
||||
OPERATOR = "@dongar:dongar.net" # an allowlisted Matrix operator in nisse-test.yaml
|
||||
DEADLINE_S = 1200
|
||||
MAX_ANSWERS = 5 # per wait for something else, before giving up
|
||||
BUG_GRACE_S = 20
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -69,6 +71,8 @@ def running(
|
||||
state_db=keep / "state.db",
|
||||
workspace=Path(f"/tmp/nisse-e2e-{run}/sandbox"),
|
||||
build_host=build_host(ssh_host),
|
||||
# Short, so a bug's grace period runs out within the test.
|
||||
tuning=replace(live_config.tuning, loop=LoopTuning(bug_grace_s=BUG_GRACE_S)),
|
||||
)
|
||||
transport = InMemoryTransport()
|
||||
monkeypatch.setattr("nisse_dev.loop.build.MatrixTransport", lambda cfg: transport)
|
||||
@@ -143,8 +147,9 @@ class Operator:
|
||||
self.transport.receive(OPERATOR, text, thread=thread)
|
||||
self.settle()
|
||||
|
||||
def drive_until(self, wait: str, issue: int = 0) -> Any:
|
||||
"""Answer whatever the agent asks until it waits for ``wait``."""
|
||||
def drive_until(self, wait: str, issue: int = 0, *, go: bool = True) -> Any:
|
||||
"""Answer whatever the agent asks until it waits for ``wait``.
|
||||
Without ``go``, a restatement is left for its grace period."""
|
||||
deadline = time.monotonic() + DEADLINE_S
|
||||
answers = 0
|
||||
while time.monotonic() < deadline:
|
||||
@@ -155,7 +160,13 @@ class Operator:
|
||||
return item
|
||||
if item.status == "closed":
|
||||
raise AssertionError(f"closed early: {self.last()}")
|
||||
if item.position == WAITING and loop.wait == GO_AHEAD:
|
||||
if wait == GO_AHEAD and loop.started:
|
||||
raise AssertionError(f"went past the go-ahead: {self.last()}")
|
||||
if item.position == WAITING and loop.wait == GO_AHEAD and not go:
|
||||
assert loop.grace_until, f"no grace period: {self.last()}"
|
||||
time.sleep(2)
|
||||
self.tick() # what the agent's timer would do
|
||||
elif item.position == WAITING and loop.wait == GO_AHEAD:
|
||||
self.say("go", issue)
|
||||
elif item.position == WAITING and loop.wait == ANSWER:
|
||||
answers += 1
|
||||
@@ -310,3 +321,107 @@ def test_an_issue_blocked_on_another_waits_for_it(
|
||||
assert f"\n+def vowels_{run}" not in diff
|
||||
finally:
|
||||
operator.tidy(issues, prs)
|
||||
|
||||
|
||||
def label(gitea: GiteaClient, repo: str, name: str) -> int:
|
||||
"""The repository's label ``name``, made if it's missing."""
|
||||
for existing in gitea._request("GET", f"/repos/{repo}/labels"):
|
||||
if existing["name"] == name:
|
||||
return int(existing["id"])
|
||||
made = gitea._request(
|
||||
"POST", f"/repos/{repo}/labels", json={"name": name, "color": "#ee0701"}
|
||||
)
|
||||
return int(made["id"])
|
||||
|
||||
|
||||
def test_a_bug_goes_ahead_after_its_grace_period(
|
||||
running: tuple[RunningLoop, InMemoryTransport, str],
|
||||
) -> None:
|
||||
"""M2.3: a `bug` issue is reproduced and restated, and work starts when
|
||||
the grace period runs out, with no go from the operator."""
|
||||
agent, transport, run = running
|
||||
operator = Operator(agent, transport)
|
||||
repo, base = agent.config.gitea_repo, agent.config.base_branch
|
||||
as_operator = operator_gitea(agent.config.gitea_url)
|
||||
issues: list[int] = []
|
||||
prs: list[int] = []
|
||||
try:
|
||||
# A real bug on this run's base branch, for the agent to reproduce.
|
||||
code = (
|
||||
f"def last_word_{run}(text):\n"
|
||||
' """The last word of text, or "" if it has none."""\n'
|
||||
" return text.split()[-1]\n"
|
||||
)
|
||||
as_operator._request(
|
||||
"POST",
|
||||
f"/repos/{repo}/contents/sandbox/last_word_{run}.py",
|
||||
json={
|
||||
"branch": base,
|
||||
"content": base64.b64encode(code.encode()).decode(),
|
||||
"message": f"last_word_{run}, with a bug",
|
||||
},
|
||||
)
|
||||
data = as_operator._request(
|
||||
"POST",
|
||||
f"/repos/{repo}/issues",
|
||||
json={
|
||||
"title": f"last_word_{run} crashes on empty text",
|
||||
"body": f"`last_word_{run}('')` in sandbox/last_word_{run}.py "
|
||||
'raises IndexError; it should return "" as its docstring says.',
|
||||
"labels": [label(as_operator, repo, "bug")],
|
||||
"assignees": [agent.account],
|
||||
},
|
||||
)
|
||||
issues.append(int(data["number"]))
|
||||
operator.tick()
|
||||
assert loop_of(operator.item(issues[0])).labels == ("bug",)
|
||||
item = operator.drive_until(MERGE, issues[0], go=False)
|
||||
prs.append(loop_of(item).pr)
|
||||
orchestrator = agent.worker.call(
|
||||
lambda: agent.store.transcript(item.id, "orchestrator").read()
|
||||
)
|
||||
said = [getattr(e.event, "text", "") for e in orchestrator]
|
||||
assert any(t.endswith("from the bug grace period") for t in said), said
|
||||
finally:
|
||||
operator.tidy(issues, prs)
|
||||
|
||||
|
||||
def test_a_plain_yes_is_a_go_ahead(
|
||||
running: tuple[RunningLoop, InMemoryTransport, str],
|
||||
) -> None:
|
||||
"""M2.3: "yep, that's right" to a restatement is read as a go-ahead,
|
||||
and the agent says so before acting."""
|
||||
agent, transport, run = running
|
||||
operator = Operator(agent, transport)
|
||||
repo = agent.config.gitea_repo
|
||||
as_operator = operator_gitea(agent.config.gitea_url)
|
||||
issues: list[int] = []
|
||||
prs: list[int] = []
|
||||
try:
|
||||
issues.append(as_operator.create_issue(repo, f"vowels_{run}", task(run)))
|
||||
as_operator._request(
|
||||
"PATCH",
|
||||
f"/repos/{repo}/issues/{issues[0]}",
|
||||
json={"assignees": [agent.account]},
|
||||
)
|
||||
operator.tick()
|
||||
for _ in range(MAX_ANSWERS):
|
||||
operator.drive_until(GO_AHEAD, issues[0])
|
||||
if "?" not in operator.last():
|
||||
break
|
||||
# A restatement with a question in it can't be agreed to as it
|
||||
# stands: answer first (the reader rightly says no to a yes).
|
||||
operator.say(
|
||||
f"Not yet: a new function vowels_{run}, not reusing anything "
|
||||
"that exists. Put up a new restatement with that.",
|
||||
issues[0],
|
||||
)
|
||||
operator.say("Yep, that's right, sounds good.", issues[0])
|
||||
said = [m.text for m in transport.outbound]
|
||||
assert any(t.startswith("Taking that as a go-ahead on rev-") for t in said)
|
||||
assert loop_of(operator.item(issues[0])).started
|
||||
pr = loop_of(operator.item(issues[0])).pr
|
||||
if pr:
|
||||
prs.append(pr)
|
||||
finally:
|
||||
operator.tidy(issues, prs)
|
||||
Reference in new issue
Block a user