Plan mode proposes, and you decide whether to carry it out
`plan_submit` records an ordered set of steps and ends the turn. Offered in Plan mode and nowhere else: it stops the reply, and a model in Auto mode proposing a plan instead of doing the work would be obeying the wrong instinct at the worst moment. The plan is stored on the message rather than parsed back out of the prose, so the button sends exactly what was proposed. It gets one more request to say what it proposed and why -- a bubble containing only a card reads as though the model had nothing to add -- but with the tools withdrawn, so "one more round" cannot become three rounds of it changing its mind about a plan somebody is being asked to approve. Carrying it out switches to Edit, never Auto. The plan was written under a mode where every command stopped for approval, and a button that also removed the asking is not the button anybody pressed. It goes back quoted and attributed, not stated: a plan whose text came out of a file the model read must not arrive in the most trusted role in the transcript wearing the reader's authority. Also closes the rewind gap. Editing or regenerating a turn rewinds the transcript and not the machine, so `rewound_at` is stamped and the harness says so. Nothing tries to undo anything out there -- the project directory is somebody's real working tree, and deleting their work to match a rewound transcript would be far worse than the inconsistency. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
@@ -8,6 +8,7 @@ than a stub that always agrees.
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json as _json
|
||||
|
||||
import pytest
|
||||
|
||||
@@ -483,3 +484,170 @@ async def test_an_agent_chat_is_told_its_real_round_budget(db, user_id, machine)
|
||||
db, user, tools_service.resolve_tools(db, chat, user).schemas, chat
|
||||
)
|
||||
assert values["max_rounds"] == "25"
|
||||
|
||||
|
||||
# --- Plan mode's artifact ---------------------------------------------------------
|
||||
def test_plan_submit_is_offered_only_in_plan_mode(db, user_id, machine):
|
||||
"""It ends the reply. A model in Auto mode that proposed a plan instead of
|
||||
doing the work would be obeying the wrong instinct at the wrong moment."""
|
||||
user = db.get(User, user_id)
|
||||
planning, _p = _setup(db, user_id, machine, mode=policy.MODE_PLAN)
|
||||
assert "plan_submit" in _offered(db, planning, user)
|
||||
|
||||
planning.agent_mode = policy.MODE_AUTO
|
||||
db.commit()
|
||||
assert "plan_submit" not in _offered(db, planning, user)
|
||||
|
||||
|
||||
async def test_a_plan_ends_the_reply_and_lands_on_the_message(
|
||||
db, user_id, machine, monkeypatch
|
||||
):
|
||||
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_PLAN)
|
||||
message_id = _pending_reply(db, chat)
|
||||
|
||||
plan = {"title": "Tidy the logs", "steps": ["Read the log", "Rotate it", "Restart"]}
|
||||
rounds = [
|
||||
[_chunk("plan_submit", _json.dumps(plan))],
|
||||
# The model gets one wordless round to explain itself. If it tries to
|
||||
# act in it -- as this one does -- there are no tools to act with.
|
||||
[_chunk("shell_run", '{"command": "rm -rf /"}'), _text("Here is what I would do.")],
|
||||
]
|
||||
seen: list[dict] = []
|
||||
monkeypatch.setattr(generation_service, "stream_chat", _stub_stream(rounds, seen))
|
||||
|
||||
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
||||
await asyncio.wait_for(generation_service._run(generation), timeout=10)
|
||||
|
||||
assert generation.plan == plan
|
||||
assert len(seen) == 2, "one round to plan, one to say what it proposed"
|
||||
assert "tools" not in seen[1], "the second round is offered nothing to act with"
|
||||
assert len(generation.tool_events) == 1, "the shell call had no tool to reach"
|
||||
|
||||
db.expire_all()
|
||||
message = db.get(Message, message_id)
|
||||
assert message.plan_json == plan
|
||||
|
||||
|
||||
async def test_a_plan_with_no_steps_is_sent_back(db, user_id, machine, monkeypatch):
|
||||
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_PLAN)
|
||||
message_id = _pending_reply(db, chat)
|
||||
monkeypatch.setattr(
|
||||
generation_service,
|
||||
"stream_chat",
|
||||
_stub_stream(
|
||||
[[_chunk("plan_submit", '{"title": "Nothing", "steps": []}')], [_text("Sorry.")]],
|
||||
[],
|
||||
),
|
||||
)
|
||||
|
||||
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
||||
await asyncio.wait_for(generation_service._run(generation), timeout=10)
|
||||
|
||||
assert generation.plan is None
|
||||
assert generation.tool_events[0]["status"] == "error"
|
||||
|
||||
|
||||
# --- Carrying a plan out ------------------------------------------------------------
|
||||
def test_executing_a_plan_switches_to_edit_and_quotes_it(client, db, registered, machine):
|
||||
"""Edit, never Auto. The plan was written under a mode where every command
|
||||
stopped for approval, and a button that also removed the asking is not the
|
||||
button anybody pressed."""
|
||||
from sqlalchemy import select as _select
|
||||
|
||||
user = db.scalar(_select(User))
|
||||
chat, _profile = _setup(db, user.id, machine, mode=policy.MODE_PLAN)
|
||||
message = Message(
|
||||
chat_id=chat.id,
|
||||
role=ROLE_ASSISTANT,
|
||||
content="Here is what I would do.",
|
||||
complete=True,
|
||||
plan_json={"title": "Tidy the logs", "steps": ["Read the log", "Rotate it"]},
|
||||
)
|
||||
db.add(message)
|
||||
db.commit()
|
||||
|
||||
response = client.post(f"/api/chats/{chat.id}/messages/{message.id}/execute-plan")
|
||||
assert response.status_code == 200
|
||||
|
||||
db.refresh(chat)
|
||||
assert chat.agent_mode == policy.MODE_EDIT
|
||||
|
||||
sent = db.scalars(
|
||||
_select(Message).where(Message.chat_id == chat.id, Message.role == "user")
|
||||
).all()[-1]
|
||||
assert "Tidy the logs" in sent.content
|
||||
assert "Read the log" in sent.content
|
||||
# Quoted rather than stated. A plan whose text came out of a file the model
|
||||
# read must not arrive wearing the reader's authority.
|
||||
assert sent.content.lstrip().startswith("Carry out the plan you proposed above")
|
||||
assert "> " in sent.content
|
||||
|
||||
|
||||
def test_a_message_with_no_plan_cannot_be_executed(client, db, registered, machine):
|
||||
from sqlalchemy import select as _select
|
||||
|
||||
user = db.scalar(_select(User))
|
||||
chat, _profile = _setup(db, user.id, machine)
|
||||
message = Message(chat_id=chat.id, role=ROLE_ASSISTANT, content="hi", complete=True)
|
||||
db.add(message)
|
||||
db.commit()
|
||||
|
||||
assert (
|
||||
client.post(f"/api/chats/{chat.id}/messages/{message.id}/execute-plan").status_code
|
||||
== 404
|
||||
)
|
||||
|
||||
|
||||
# --- Rewind ---------------------------------------------------------------------------
|
||||
def test_editing_a_turn_records_that_the_machine_did_not_rewind(
|
||||
client, db, registered, machine
|
||||
):
|
||||
"""The transcript goes back; the project directory does not. Deleting
|
||||
somebody's real working tree to match would be far worse than the
|
||||
inconsistency, so the model is told instead."""
|
||||
from sqlalchemy import select as _select
|
||||
|
||||
user = db.scalar(_select(User))
|
||||
chat, _profile = _setup(db, user.id, machine)
|
||||
first = Message(chat_id=chat.id, role="user", content="do a thing", complete=True)
|
||||
db.add(first)
|
||||
db.commit()
|
||||
|
||||
assert chat.rewound_at is None
|
||||
client.post(
|
||||
f"/api/chats/{chat.id}/messages/{first.id}/edit", data={"content": "do another thing"}
|
||||
)
|
||||
db.refresh(chat)
|
||||
assert chat.rewound_at is not None
|
||||
|
||||
|
||||
def test_an_ordinary_chat_records_no_rewind(client, db, registered, machine):
|
||||
from sqlalchemy import select as _select
|
||||
|
||||
user = db.scalar(_select(User))
|
||||
chat, _profile = _setup(db, user.id, machine, kind="chat")
|
||||
first = Message(chat_id=chat.id, role="user", content="hello", complete=True)
|
||||
db.add(first)
|
||||
db.commit()
|
||||
|
||||
client.post(f"/api/chats/{chat.id}/messages/{first.id}/edit", data={"content": "hi"})
|
||||
db.refresh(chat)
|
||||
assert chat.rewound_at is None
|
||||
|
||||
|
||||
async def test_the_harness_warns_after_a_rewind(db, user_id, machine):
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from lembas.services import harness
|
||||
|
||||
chat, _profile = _setup(db, user_id, machine)
|
||||
user = db.get(User, user_id)
|
||||
offered = tools_service.resolve_tools(db, chat, user).schemas
|
||||
|
||||
assert "was rewound" not in harness.compose(db, user, offered, chat)
|
||||
|
||||
chat.rewound_at = datetime.now(UTC)
|
||||
db.commit()
|
||||
text = harness.compose(db, user, offered, chat)
|
||||
assert "was rewound" in text
|
||||
assert "still there" in text
|
||||
|
||||
Reference in New Issue
Block a user