diff --git a/src/lembas/api/canvas.py b/src/lembas/api/canvas.py new file mode 100644 index 0000000..a28307e --- /dev/null +++ b/src/lembas/api/canvas.py @@ -0,0 +1,198 @@ +"""The canvas panel: open a file, read it, change it, save it. + +Every route answers with an HTML fragment, errors included. An exception page +swapped into a side panel is a blank side panel, and a panel that goes blank +tells somebody nothing about why. + +`GET` never moves the active tab. There is no CSRF token in this application and +the session cookie is SameSite Lax, so a state-changing GET is a link somebody +can be made to follow -- and one of the things a tab can be is a file on +somebody's server. +""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Form, HTTPException, Request, Response, status +from sqlalchemy.orm import Session as DBSession + +from lembas.api.deps import Db, RequiredUser +from lembas.db.models import Chat, User +from lembas.services import canvas as canvas_service +from lembas.services import generation as generation_service +from lembas.services.agent.base import Conflict +from lembas.services.markdown import highlight_code, render_markdown +from lembas.web.templating import templates + +log = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/chats", tags=["canvas"]) + + +def _owned_chat(db: DBSession, chat_id: str, user_id: str) -> Chat: + """404 rather than 403 for somebody else's chat: whether it exists at all is + not this account's business.""" + chat = db.get(Chat, chat_id) + if chat is None or chat.user_id != user_id: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That chat no longer exists.") + return chat + + +async def _panel( + request: Request, + db: DBSession, + user: User, + chat: Chat, + *, + key: str = "", + message: str = "", + conflict: canvas_service.Doc | None = None, + mine: str = "", +) -> Response: + """The strip and whichever tab is in front, as one fragment. + + Both together, always. Rendering only the body would leave the strip showing + a tab that is no longer there after a close, and rendering only the strip + would leave the previous file on screen after a switch. + """ + wanted = key or canvas_service.active_of(chat) + doc: canvas_service.Doc | None = None + error = message + if wanted and not error: + try: + doc = await canvas_service.load(db, user, chat, wanted) + except canvas_service.Refused as exc: + error = str(exc) + except Exception: # pragma: no cover - a machine going away mid-request + log.exception("canvas could not open %s", wanted) + error = "That could not be opened." + + body = "" + if doc is not None and doc.text: + # The one `|safe` in this panel, and it is safe because pygments escapes + # what it is given. Markdown goes through render_markdown, the single + # path in this application allowed to emit HTML. Everything else -- the + # editor's contents, the titles, the paths -- is escaped by Jinja. + body = render_markdown(doc.text) if doc.markdown else highlight_code(doc.text, doc.language) + + return templates.TemplateResponse( + request, + "chat/_canvas_inner.html", + { + "user": user, + "chat": chat, + "tabs": canvas_service.tabs_of(chat), + "active": wanted, + "doc": doc, + "rendered": body, + "error": error, + "conflict": conflict, + "mine": mine, + "canvas_agent": canvas_service.agent_ready(db, user, chat) is not None, + }, + ) + + +@router.get("/{chat_id}/canvas") +async def show(request: Request, db: Db, user: RequiredUser, chat_id: str, key: str = ""): + """Whatever is in front, or the tab named by `?key=`. + + Read-only in every sense: a `?key=` that is not open does not become open, + it is simply shown. Opening is a POST. + """ + chat = _owned_chat(db, chat_id, user.id) + return await _panel(request, db, user, chat, key=key) + + +@router.post("/{chat_id}/canvas/tabs") +async def open_tab( + request: Request, + db: Db, + user: RequiredUser, + chat_id: str, + key: str = Form(...), + title: str = Form(""), +): + """Open a file, or bring an already-open one to the front. + + Idempotent, because opening what is already open is switching to it -- the + same reason `generation.ensure` is idempotent. + """ + chat = _owned_chat(db, chat_id, user.id) + + try: + doc = await canvas_service.load(db, user, chat, key) + except canvas_service.Refused as exc: + return await _panel(request, db, user, chat, message=str(exc)) + + state = canvas_service.open_tab( + dict(chat.canvas_json or {}), + {"key": doc.key, "title": title.strip() or doc.title, "source": doc.key.split(":")[0]}, + ) + # Reassigned rather than mutated: an in-place edit of a JSON column is not + # reliably detected as a change. + chat.canvas_json = state + db.commit() + + # A reply running right now holds its own snapshot, seeded when it started. + # Without this the next frame it sends would contradict what was just + # swapped in -- the same reach into live state `request_stop` makes. + live = generation_service.running_for(chat.id) + if live is not None: + canvas_service.open_tab(live.canvas, {"key": doc.key, "title": doc.title}) + + return await _panel(request, db, user, chat, key=doc.key) + + +@router.post("/{chat_id}/canvas/tabs/close") +async def close_tab( + request: Request, db: Db, user: RequiredUser, chat_id: str, key: str = Form(...) +): + chat = _owned_chat(db, chat_id, user.id) + chat.canvas_json = canvas_service.close_tab(dict(chat.canvas_json or {}), key) + db.commit() + + live = generation_service.running_for(chat.id) + if live is not None: + canvas_service.close_tab(live.canvas, key) + + return await _panel(request, db, user, chat) + + +@router.post("/{chat_id}/canvas/save") +async def save( + request: Request, + db: Db, + user: RequiredUser, + chat_id: str, + key: str = Form(...), + text: str = Form(""), + revision: str = Form(""), +): + """Write it back. + + A conflict comes back as a card, at 200, so htmx swaps it: the panel has to + be able to show Overwrite, Discard mine and Show what changed, and none of + those can be offered from an error status htmx will not render. Never save + silently over a change; never discard silently either. + """ + chat = _owned_chat(db, chat_id, user.id) + + try: + await canvas_service.save(db, user, chat, key, text, revision) + except Conflict: + try: + theirs = await canvas_service.load(db, user, chat, key) + except canvas_service.Refused as exc: + return await _panel(request, db, user, chat, key=key, message=str(exc)) + return await _panel(request, db, user, chat, key=key, conflict=theirs, mine=text) + except canvas_service.Refused as exc: + return await _panel(request, db, user, chat, key=key, message=str(exc)) + except Exception: # pragma: no cover - the machine going away mid-write + log.exception("canvas could not save %s", key) + return await _panel( + request, db, user, chat, key=key, message="That could not be saved." + ) + + return await _panel(request, db, user, chat, key=key) diff --git a/src/lembas/api/chats.py b/src/lembas/api/chats.py index 1a29379..36fe18c 100644 --- a/src/lembas/api/chats.py +++ b/src/lembas/api/chats.py @@ -8,6 +8,7 @@ import logging import time from collections.abc import AsyncIterator from datetime import UTC, datetime +from types import SimpleNamespace from fastapi import APIRouter, Depends, Form, HTTPException, Request, status from fastapi.responses import HTMLResponse, JSONResponse, Response, StreamingResponse @@ -855,6 +856,25 @@ def _ask_html(chat_id: str, pending) -> str: ) +def _canvas_tabs(chat_id: str, state: dict) -> str: + """The canvas tab strip, as an out-of-band swap. + + Out of band because it belongs to a panel, not to the bubble the stream is + writing into -- the same move the `done` frame already makes for the chat + title. Only the strip: pushing the file's contents on every version bump + would be a lot of bytes for nothing, and would overwrite a textarea somebody + is typing in. The active tab's body fetches itself once instead. + """ + return templates.get_template("chat/_canvas_tabs.html").render( + { + "chat": SimpleNamespace(id=chat_id), + "tabs": state.get("tabs") or [], + "active": state.get("active") or "", + "oob": True, + } + ) + + async def _follow(chat_id: str, message_id: str) -> AsyncIterator[str]: """Stream a generation that is running independently of this request. @@ -881,6 +901,19 @@ async def _follow(chat_id: str, message_id: str) -> AsyncIterator[str]: yield sse.event("tools", _tool_activity(generation.tool_events)) if generation.content: yield sse.event("render", render_markdown(generation.text)) + if generation.canvas.get("tabs"): + # Guarded on truthiness, which puts this in the + # reasoning/tools/render group and not the + # metrics/status/ask one. Those three are sent even when + # empty *because* each has to be able to clear itself; this + # one must never be able to, since an empty canvas frame + # would close every tab somebody had open. The card that + # could be pressed twice, with the sign reversed. + # + # The whole strip each time, not a delta, so a follower + # attaching mid-reply gets every tab the reply has touched + # rather than the ones that happened to arrive after it. + yield sse.event("canvas", _canvas_tabs(chat_id, generation.canvas)) yield sse.event("metrics", _metrics_html(generation)) yield sse.event("status", escape_text(generation.status)) yield sse.event("ask", _ask_html(chat_id, generation.pending)) diff --git a/src/lembas/api/files.py b/src/lembas/api/files.py index ee7ec4f..872e16d 100644 --- a/src/lembas/api/files.py +++ b/src/lembas/api/files.py @@ -19,7 +19,7 @@ from fastapi.responses import FileResponse from sqlalchemy import select from lembas.api.deps import Db, RequiredUser, require_permission -from lembas.db.models import Attachment, Document, KnowledgeBase, Note +from lembas.db.models import Attachment, Chat, Document, KnowledgeBase, Note from lembas.security import permissions from lembas.services import files as files_service from lembas.services import settings_store @@ -189,6 +189,41 @@ async def attach_from_note( ) +@router.post("/from-scratch", dependencies=[Depends(require_permission("files.upload"))]) +async def attach_from_scratch( + request: Request, db: Db, user: RequiredUser, chat_id: str = Form("") +) -> Response: + """Attach this chat's scratch document. + + A copy, like every other attach path, and here the reason is at its + sharpest: the pad goes on being written after the message is sent, by the + person and by the model, and a transcript that changed underneath itself + every time either of them typed would be no record at all. + """ + from lembas.services import scratch as scratch_service + + chat = db.get(Chat, chat_id) if chat_id else None + if chat is None or chat.user_id != user.id: + return _not_available(request, "scratch document") + + doc = scratch_service.get(db, chat) + if doc is None or not (doc.body or "").strip(): + return _not_available(request, "scratch document") + + return _chip( + request, + files_service.store_text( + db, + user_id=user.id, + chat_id=chat.id, + filename=f"{doc.title or 'scratch'}.md", + text=doc.body, + source_path=doc.title or "Scratch", + source_label="Scratch", + ), + ) + + @router.post("/from-skill", dependencies=[Depends(require_permission("files.upload"))]) async def attach_from_skill( request: Request, db: Db, user: RequiredUser, skill_id: str = Form(""), chat_id: str = Form("") diff --git a/src/lembas/api/pages.py b/src/lembas/api/pages.py index 1244ff9..138f53b 100644 --- a/src/lembas/api/pages.py +++ b/src/lembas/api/pages.py @@ -11,6 +11,7 @@ from lembas.api.deps import Db, RequiredUser from lembas.db.models import KIND_CHAT, KINDS, Chat, Folder, KnowledgeBase, Message, User from lembas.security import permissions from lembas.services import audio as audio_service +from lembas.services import canvas as canvas_service from lembas.services import chat as chat_service from lembas.services import compaction as compaction_service from lembas.services import settings_store @@ -185,6 +186,16 @@ def _agent_context(db: DBSession, user: User, chat: Chat | None) -> dict: for m in agent_policy.MODES ], "terminal_enabled": _terminal_enabled(db, user, chat, current), + # Any chat that exists. Deliberately not gated the way the terminal is: + # half the canvas's sources -- notes, skills, this chat's attachments, + # its own scratch document -- need no machine at all, so the terminal's + # total gate would remove a working feature because one source is + # unavailable. Absent on the new-chat screen for the reason the scope + # menu is: there is no row yet to hang a tab on. + "canvas_enabled": chat is not None, + # And whether it may *also* reach project files. Re-derived server-side + # on every canvas request; this flag only decides what the panel offers. + "canvas_agent": canvas_service.agent_ready(db, user, chat) is not None, } diff --git a/src/lembas/api/preferences.py b/src/lembas/api/preferences.py index 8465ba3..38f392f 100644 --- a/src/lembas/api/preferences.py +++ b/src/lembas/api/preferences.py @@ -44,6 +44,7 @@ async def set_theme(db: Db, user: RequiredUser, theme: str = Body(..., embed=Tru # panel they cannot see to drag back. LAYOUT_BOUNDS = { "--terminal-width": (384, 2400), + "--canvas-width": (384, 2400), "--inspector-width": (280, 2400), "--sidebar-width": (200, 800), } diff --git a/src/lembas/db/models/__init__.py b/src/lembas/db/models/__init__.py index 91601bf..df9c91e 100644 --- a/src/lembas/db/models/__init__.py +++ b/src/lembas/db/models/__init__.py @@ -18,6 +18,7 @@ from lembas.db.models.attachment import ( KIND_TEXT, Attachment, ) +from lembas.db.models.canvas import ScratchDoc from lembas.db.models.chat import ( KIND_AGENT, KIND_CHAT, @@ -124,6 +125,7 @@ __all__ = [ "Message", "Model", "Note", + "ScratchDoc", "Session", "Setting", "Share", diff --git a/src/lembas/db/models/canvas.py b/src/lembas/db/models/canvas.py new file mode 100644 index 0000000..a7d1d3c --- /dev/null +++ b/src/lembas/db/models/canvas.py @@ -0,0 +1,50 @@ +"""A chat's own working surface.""" + +from __future__ import annotations + +from sqlalchemy import ForeignKey, String, Text +from sqlalchemy.orm import Mapped, mapped_column + +from lembas.db.base import Base, Timestamps, UUIDPrimaryKey +from lembas.db.models.library import AUTHOR_USER + + +class ScratchDoc(UUIDPrimaryKey, Timestamps, Base): + """A text artefact belonging to one chat, written by either side of it. + + The model can write into it, the person can edit it, and either can hand the + result to the next message as an ordinary attachment. Distinct from a note, + which is a durable artefact of the reader's that outlives the chat -- this + is the chat's own record of what it is working on, which is the same line + `plan_update` is on rather than `notes_edit`. + + A separate table rather than a column on `chats` for one plain reason: + `select(Chat)` runs for the sidebar on every page load, and SQLAlchemy loads + every column -- so a Text body would ride along with two hundred sidebar + rows to answer a question about none of them. + + One per chat. Several would mean a picker, names, deletion and a sweep, and + would mean the model choosing an id; one means `scratch:` is + derivable rather than looked up. If several are ever wanted, they are notes. + """ + + __tablename__ = "scratch_docs" + + chat_id: Mapped[str] = mapped_column( + String(32), + ForeignKey("chats.id", ondelete="CASCADE"), + nullable=False, + index=True, + unique=True, + ) + user_id: Mapped[str] = mapped_column( + String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + title: Mapped[str] = mapped_column(String(300), default="Scratch") + body: Mapped[str] = mapped_column(Text, default="") + # Who wrote it last, so the panel can say. Not authorisation: the chat's + # owner is the only person who can reach it either way. + author: Mapped[str] = mapped_column(String(16), default=AUTHOR_USER, nullable=False) + + def __repr__(self) -> str: + return f"" diff --git a/src/lembas/db/models/chat.py b/src/lembas/db/models/chat.py index f4cff42..4eaabaf 100644 --- a/src/lembas/db/models/chat.py +++ b/src/lembas/db/models/chat.py @@ -218,6 +218,18 @@ class Chat(UUIDPrimaryKey, Timestamps, Base): # representations of "on" makes "why is this off?" unanswerable. scope_json: Mapped[dict[str, Any]] = mapped_column(JSONDict, default=dict) + # Which files are open in the canvas panel, and which of them is in front. + # {"tabs": [{"key": "agent:/srv/app/main.py", "title": …, "source": …}], + # "active": "agent:/srv/app/main.py"} + # + # Server-side rather than in the browser because a model reading a file + # opens a tab, and every frame this application streams is HTML swapped + # whole -- if the browser owned the list, the server could not render the + # strip and the frame would have to become data for JavaScript to interpret. + # One chat, one canvas, the same consequence the terminal panel documents: + # two tabs on the same chat share it. + canvas_json: Mapped[dict[str, Any]] = mapped_column(JSONDict, default=dict) + # --- Compaction ---------------------------------------------------------- # A summary of the turns up to `compacted_through_id`, sent in their place. # The messages themselves are kept and still shown; they simply stop being diff --git a/src/lembas/main.py b/src/lembas/main.py index fc86502..54fac6f 100644 --- a/src/lembas/main.py +++ b/src/lembas/main.py @@ -25,6 +25,7 @@ from lembas.api import ( agents, audio, auth, + canvas, chats, files, folders, @@ -134,6 +135,7 @@ def create_app() -> FastAPI: app.include_router(auth.router) app.include_router(preferences.router) app.include_router(chats.router) + app.include_router(canvas.router) app.include_router(terminal.router) app.include_router(audio.router) app.include_router(files.router) diff --git a/src/lembas/security/permissions.py b/src/lembas/security/permissions.py index 90af794..b9b4fba 100644 --- a/src/lembas/security/permissions.py +++ b/src/lembas/security/permissions.py @@ -145,6 +145,15 @@ PERMISSION_DEFS: tuple[PermissionDef, ...] = ( True, "Chat", ), + PermissionDef( + "tools.scratch", + "Write in the canvas", + "Let a model build something up in this chat's scratch document, which " + "sits open beside the conversation and can be edited and attached to a " + "message. It belongs to the chat and is not searchable afterwards.", + True, + "Chat", + ), PermissionDef( "audio.transcribe", "Dictate messages", diff --git a/src/lembas/services/agent/base.py b/src/lembas/services/agent/base.py index 0f879ba..a971161 100644 --- a/src/lembas/services/agent/base.py +++ b/src/lembas/services/agent/base.py @@ -107,6 +107,55 @@ class RemoteEntry: return self.name.startswith(".") +@dataclass(frozen=True) +class RemoteFile: + """A file as somebody is about to edit it, rather than as a model reads it. + + Separate from what `read_file` returns for the same reason `RemoteEntry` is + separate from `list_dir`: the model-facing contract is right for a model and + wrong here. `read_file` runs its result through `clean_output`, which strips + escape sequences and decodes with errors="replace" -- so a file opened + through it and saved back would come out rewritten. + + `binary` means there is nothing safe to put in a textarea, and the tab opens + read-only. `truncated` means the same for a different reason: saving back + the first 256KB of a larger file is how the rest of it is deleted. + """ + + text: str + size: int = 0 + mtime: int = 0 + truncated: bool = False + binary: bool = False + + @property + def revision(self) -> str: + return revision_of(self.mtime, self.size) + + +def revision_of(mtime: int, size: int) -> str: + """An opaque token saying which version of a file was read. + + Round-tripped through a hidden field and compared on the way back in. Not a + hash: hashing means reading the whole file again on every save, and this + catches the case it exists for -- somebody else's editor, a build, a + checkout -- without it. + """ + return f"{mtime}:{size}" + + +class Conflict(Exception): + """The file moved between being opened and being saved. + + Carries the revision found instead, so the card offering Overwrite has + something to compare against. + """ + + def __init__(self, found: str = "") -> None: + super().__init__("That file changed after it was opened.") + self.found = found + + class Executor(Protocol): """How a target is acted on. See `ssh.py`; there is no local variant.""" @@ -116,6 +165,10 @@ class Executor(Protocol): async def write_file(self, path: str, text: str) -> int: ... + async def read_text(self, path: str, *, max_bytes: int) -> RemoteFile: ... + + async def write_text(self, path: str, text: str, *, if_unchanged: str) -> RemoteFile: ... + async def list_dir(self, path: str) -> list[str]: ... async def scan_dir(self, path: str) -> list[RemoteEntry]: ... diff --git a/src/lembas/services/agent/ssh.py b/src/lembas/services/agent/ssh.py index 132dbee..4c1ab78 100644 --- a/src/lembas/services/agent/ssh.py +++ b/src/lembas/services/agent/ssh.py @@ -26,17 +26,21 @@ forgot to install it gets a sentence rather than an ImportError at startup. from __future__ import annotations +import contextlib import logging import time from typing import Any from lembas.db.models import AUTH_PASSWORD, SshProfile from lembas.services.agent.base import ( + Conflict, ExecError, ExecRequest, ExecResult, RemoteEntry, + RemoteFile, clean_output, + revision_of, ) from lembas.services.crypto import decrypt @@ -284,6 +288,111 @@ class SshExecutor: raise self._wrap(exc) from exc return len(payload) + # --- The same files, for somebody about to edit them --------------------- + # Deliberately not `read_file`/`write_file`, and those two are deliberately + # left exactly as they are: what they return is a contract a model has been + # shown, and it is the right contract for a model. + # + # It is the wrong one for an editor. `read_file` ends in `clean_output`, + # which strips ANSI escape sequences and decodes with errors="replace" -- + # correct for the output of a command, and for a file it means that opening + # one containing an escape byte and pressing Save rewrites it with the + # escapes gone and every undecodable byte replaced by U+FFFD. `write_file` + # truncates at MAX_WRITE_BYTES, which a model is told about and a person + # pressing Save is not. + async def read_text(self, path: str, *, max_bytes: int = MAX_READ_BYTES) -> RemoteFile: + """A file as somebody is about to edit it. + + Strict decoding, so a file this cannot represent faithfully is reported + as binary rather than silently mangled into something that would be + saved back. The stat and the read share one connection: connections are + per call, so doing it in two is two handshakes and two authentications + to open one file. + """ + import asyncssh + + try: + async with ( + self._connect() as conn, + conn.start_sftp_client() as sftp, + sftp.open(self._resolve(path), "rb") as handle, + ): + attrs = await handle.stat() + data = await handle.read(max_bytes + 1) + except asyncssh.SFTPNoSuchFile as exc: + raise ExecError(f"There is no file at {path}.") from exc + except asyncssh.SFTPPermissionDenied as exc: + raise ExecError(f"Not allowed to read {path}.") from exc + except (OSError, asyncssh.Error) as exc: + raise self._wrap(exc) from exc + + truncated = len(data) > max_bytes + data = data[:max_bytes] + size = int(getattr(attrs, "size", None) or len(data)) + mtime = int(getattr(attrs, "mtime", None) or 0) + + # A NUL in the first few kilobytes, or anything that will not decode. + # Either way there is nothing safe to put in a textarea. + if b"\0" in data[:8192]: + return RemoteFile("", size, mtime, truncated, binary=True) + try: + text = data.decode("utf-8") + except UnicodeDecodeError: + return RemoteFile("", size, mtime, truncated, binary=True) + return RemoteFile(text, size, mtime, truncated, binary=False) + + async def write_text(self, path: str, text: str, *, if_unchanged: str = "") -> RemoteFile: + """Write a file, refusing if it moved under the editor. + + `if_unchanged` is the token `read_text` handed out. The re-stat and the + write happen on one connection, which is the narrowest window SFTP + allows; there is no compare-and-swap here and this does not pretend to + be atomic. It catches what it exists for -- another editor, a build, a + checkout between opening a tab and pressing Save -- and not a race + measured in milliseconds. + + Oversize is refused rather than truncated. `write_file` truncates + because a model is told how many bytes it wrote; somebody pressing Save + would lose the tail of their file with nothing said. + """ + import asyncssh + + payload = text.encode("utf-8") + if len(payload) > MAX_WRITE_BYTES: + raise ExecError( + f"That is {len(payload) // 1024}KB and the limit is " + f"{MAX_WRITE_BYTES // 1024}KB. Nothing was written." + ) + + target = self._resolve(path) + try: + async with self._connect() as conn, conn.start_sftp_client() as sftp: + if if_unchanged: + current = "" + with contextlib.suppress(asyncssh.SFTPNoSuchFile): + attrs = await sftp.stat(target) + current = revision_of( + int(getattr(attrs, "mtime", None) or 0), + int(getattr(attrs, "size", None) or 0), + ) + if current and current != if_unchanged: + raise Conflict(current) + async with sftp.open(target, "wb") as handle: + await handle.write(payload) + attrs = await sftp.stat(target) + except asyncssh.SFTPPermissionDenied as exc: + raise ExecError(f"Not allowed to write {path}.") from exc + except (OSError, asyncssh.Error) as exc: + raise self._wrap(exc) from exc + + return RemoteFile( + text, + len(payload), + int(getattr(attrs, "mtime", None) or 0), + truncated=False, + binary=False, + ) + async def list_dir(self, path: str = "") -> list[str]: import asyncssh diff --git a/src/lembas/services/agent/tools.py b/src/lembas/services/agent/tools.py index 30bba1e..2c645f0 100644 --- a/src/lembas/services/agent/tools.py +++ b/src/lembas/services/agent/tools.py @@ -343,6 +343,27 @@ def _path_key(agent: AgentContext, path: str) -> str: return posixpath.normpath(path) +def _canvas(agent: AgentContext, path: str) -> dict[str, str]: + """"This file should be on screen." + + Written onto the event because a runner cannot write the message row -- + `_persist` is the single writer -- so the generation loop carries it, in + exactly the way it carries a merged plan. + + The key comes from `_path_key`, the same normaliser the read-path set uses, + so a tab a model opened and a tab a person opened are one tab rather than + two spellings of the same file. + + It never brings the tab to the front; see `canvas.open_tab`. This rides on + calls the model was already making, so it costs no schema and no tokens. + """ + return { + "key": f"agent:{_path_key(agent, path)}", + "title": posixpath.basename(path) or path, + "source": "agent", + } + + def _forget_instructions(agent: AgentContext, path: str) -> None: """Drop the cached AGENTS.md when the thing just written *is* it. @@ -395,7 +416,14 @@ async def _run_read(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: return ToolOutcome( text or "(the file is empty)", - _event("file_read", agent, path, status="ok", text=text[:MAX_EVENT_CHARS]), + _event( + "file_read", + agent, + path, + status="ok", + text=text[:MAX_EVENT_CHARS], + canvas=_canvas(agent, path), + ), ) @@ -436,7 +464,14 @@ async def _run_write(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: index.forget_dir(agent.profile_id, agent.project_dir) _forget_instructions(agent, path) - event = _event("file_write", agent, path, status="ok", text=f"{written} bytes") + event = _event( + "file_write", + agent, + path, + status="ok", + text=f"{written} bytes", + canvas=_canvas(agent, path), + ) if diffable and before != content: event["diff"] = patch.render(before, content, path, max_lines=MAX_DIFF_LINES) @@ -508,7 +543,14 @@ async def _run_edit(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: # cares that it exists, that cache is a copy of what is in it. _forget_instructions(agent, path) - event = _event("file_edit", agent, path, status="ok", text=f"{written} bytes") + event = _event( + "file_edit", + agent, + path, + status="ok", + text=f"{written} bytes", + canvas=_canvas(agent, path), + ) if diffable: event["diff"] = patch.render(before, after, path, max_lines=MAX_DIFF_LINES) diff --git a/src/lembas/services/canvas.py b/src/lembas/services/canvas.py new file mode 100644 index 0000000..2b7c2e5 --- /dev/null +++ b/src/lembas/services/canvas.py @@ -0,0 +1,480 @@ +"""What is open in the canvas panel, and where its contents come from. + +Six sources behind one shape. A tab key is `":"` and every source +answers the same two questions -- load this, and save that -- through one table. +A table rather than six branches for the reason `tool_labels.py` and +`sharing.RESOURCE_TYPES` are tables: six independently written permission checks +is how one of them ends up written slightly differently, and the way *that* +failure shows up is somebody editing somebody else's note. + +The panel is a person's own hands. A save on an `agent:` tab therefore does not +go through `agent/policy.py`, exactly as the terminal panel and the directory +browser do not: whoever owns the credential could write the file with `scp`. +This is the first of those exceptions that *writes*, which is worth saying out +loud -- Manual mode's "everything is shown to you before it happens" is a promise +about the model, not about the interface. +""" + +from __future__ import annotations + +import posixpath +from dataclasses import dataclass +from datetime import UTC + +from sqlalchemy.orm import Session as DBSession + +from lembas.db.models import KIND_AGENT, Attachment, Chat, SshProfile, User +from lembas.security import permissions +from lembas.services import scratch as scratch_service +from lembas.services import settings_store, sharing +from lembas.services.agent import index as index_service +from lembas.services.agent import instructions as instructions_service +from lembas.services.agent import ssh as ssh_service +from lembas.services.agent.base import Conflict, ExecError, revision_of +from lembas.services.library import documents as documents_service +from lembas.services.library import notes as notes_service +from lembas.services.library import skills as skills_service + +# How many tabs a chat keeps. A model in a long reply reads forty files, and an +# unbounded strip is a strip nobody can read -- and it would live on the chat +# row forever. Past this the oldest tab that is not in front is dropped. +MAX_TABS = 12 + +SOURCE_AGENT = "agent" +SOURCE_NOTE = "note" +SOURCE_SKILL = "skill" +SOURCE_DOC = "doc" +SOURCE_FILE = "file" +SOURCE_SCRATCH = "scratch" + + + +class Refused(Exception): + """This person may not have this, or it is not there any more. + + One exception for every source, because the panel answers all of them the + same way: a fragment saying so, in the tab, rather than an error page + swapped into the middle of a chat. + """ + + +@dataclass(frozen=True) +class Doc: + """One open file, whatever it actually is underneath.""" + + key: str + title: str + subtitle: str = "" + text: str = "" + # An opaque token saying which version this was read at, round-tripped + # through a hidden field so a save can refuse a file that moved underneath. + revision: str = "" + writable: bool = False + # A filename or close enough, for choosing a lexer. + language: str = "" + markdown: bool = False + truncated: bool = False + binary: bool = False + + @property + def editable(self) -> bool: + """Whether the box is offered at all. + + Not the same as `writable`. Saving back the first 256KB of a larger file + is how the rest of it is deleted, and a binary file has nothing safe to + put in a textarea -- both open read-only however the permissions read. + """ + return self.writable and not self.truncated and not self.binary + + +def path_key(project_dir: str, path: str) -> str: + """One name for one file, so `./a.py` and `a.py` open the same tab. + + The same normalisation `agent/tools.py:_path_key` applies to the read-path + set, and lifted here so the two cannot disagree: a tab a model opened and a + tab a person opened have to be one tab, or the panel shows the same file + twice and only one of them is the one being saved. + """ + if not posixpath.isabs(path) and project_dir: + path = posixpath.join(project_dir, path) + return posixpath.normpath(path) + + +def split(key: str) -> tuple[str, str]: + """`"agent:/srv/a:b.py"` -> `("agent", "/srv/a:b.py")`. + + `partition`, not `split`: a path may contain a colon, and a key that lost + half its path would silently open the wrong file. + """ + source, _, ref = (key or "").partition(":") + return source, ref + + +# --- The tab strip --------------------------------------------------------------- +def tabs_of(chat: Chat) -> list[dict]: + return list((chat.canvas_json or {}).get("tabs") or []) + + +def active_of(chat: Chat) -> str: + return str((chat.canvas_json or {}).get("active") or "") + + +def open_tab(state: dict, tab: dict, *, activate: bool = True) -> dict: + """Add a tab, and optionally bring it to the front. Mutates `state`. + + Mutating rather than returning a copy because the generation loop folds + several of these into one snapshot within a round: two `file_read` calls + that each read the state and wrote it back would leave only the second. + That is the lost update `plan_update` documents, in a different place. + + `activate=False` is what a *model* opening a tab does, and it is the whole + of how this feature avoids being infuriating. An agent reads forty files in + a long reply; if each one took the panel, somebody reading the third would + be dragged through the other thirty-seven, and anybody halfway through an + edit would lose it. So the model fills the strip and the person decides + what is in front. A tab they open themselves activates, because opening + something and not being shown it is the opposite failure. + """ + key = str(tab.get("key") or "") + if not key: + return state + + tabs = [t for t in (state.get("tabs") or []) if t.get("key") != key] + tabs.append({ + "key": key, + "title": str(tab.get("title") or key)[:120], + "source": str(tab.get("source") or split(key)[0]), + }) + + # Evict from the front, and never the tab in front or the one just opened. + # A model reading its way through a project must not close the file + # somebody is looking at. + keep = {key, str(state.get("active") or "")} + while len(tabs) > MAX_TABS: + victim = next((t for t in tabs if t["key"] not in keep), None) + if victim is None: + break + tabs.remove(victim) + + state["tabs"] = tabs + if activate or not state.get("active"): + # Not activating an empty panel would leave tabs with nothing in front, + # which reads as a panel that failed to load. + state["active"] = key + return state + + +def close_tab(state: dict, key: str) -> dict: + tabs = [t for t in (state.get("tabs") or []) if t.get("key") != key] + state["tabs"] = tabs + if state.get("active") == key: + state["active"] = tabs[-1]["key"] if tabs else "" + return state + + +def merge(stored: dict | None, live: dict | None) -> dict: + """Fold a reply's tabs into whatever the row says now. + + A union rather than an overwrite. `_persist` is the single writer, and the + snapshot it holds was taken when the reply began -- so overwriting would + drop a tab the person opened by hand while the reply was running. + """ + state = { + "tabs": list((stored or {}).get("tabs") or []), + "active": (stored or {}).get("active") or "", + } + for tab in (live or {}).get("tabs") or []: + # Never activating: what the row says is in front is what the person + # last chose, and a reply that finishes ten minutes later must not move + # it. The reply's own `active` is deliberately not consulted. + open_tab(state, tab, activate=False) + return state + + +# --- Which sources this chat may reach --------------------------------------------- +def agent_ready(db: DBSession, user: User, chat: Chat | None) -> SshProfile | None: + """The profile an `agent:` tab would use, or None. + + Everything `_terminal_enabled` checks except `agent.terminal`. Reading and + writing project files is what `tools.agent` is named after, and somebody who + may have a model write a file may certainly write one themselves. + + Re-derived on every request. The template flag of the same name is + decoration; this is the control. + """ + if chat is None or chat.kind != KIND_AGENT or not chat.ssh_profile_id: + return None + if not permissions.has(db, user, "tools.agent"): + return None + if not settings_store.agents(db).get("enabled"): + return None + if ssh_service.available() != "": + return None + profile = db.get(SshProfile, chat.ssh_profile_id) + if profile is None or profile.owner_id != user.id or not profile.enabled: + return None + if not profile.host_key: + return None + return profile + + +def _executor(db: DBSession, user: User, chat: Chat) -> ssh_service.SshExecutor: + profile = agent_ready(db, user, chat) + if profile is None: + raise Refused( + "This chat has no connection you can reach. Check the connection's " + "host key on the Connections page if it has not been accepted yet." + ) + return ssh_service.SshExecutor(ssh_service.spec_from(profile), chat.project_dir) + + +# --- Loading ------------------------------------------------------------------------ +async def _load_agent(db: DBSession, user: User, chat: Chat, ref: str) -> Doc: + executor = _executor(db, user, chat) + path = path_key(chat.project_dir, ref) + try: + found = await executor.read_text(path) + except ExecError as exc: + raise Refused(str(exc)) from exc + + return Doc( + key=f"{SOURCE_AGENT}:{path}", + title=posixpath.basename(path) or path, + subtitle=path, + text=found.text, + revision=found.revision, + writable=True, + language=posixpath.basename(path), + markdown=path.lower().endswith((".md", ".markdown")), + truncated=found.truncated, + binary=found.binary, + ) + + +async def _load_note(db: DBSession, user: User, chat: Chat, ref: str) -> Doc: + _needs_library(db, user) + note = notes_service.get(db, ref, user) + if note is None: + raise Refused("That note is not there any more.") + return Doc( + key=f"{SOURCE_NOTE}:{note.id}", + title=note.title or "Note", + subtitle="Note", + text=note.body or "", + revision=_stamp(note, note.body or ""), + writable=sharing.can_write(note, user), + language="note.md", + markdown=True, + ) + + +async def _load_skill(db: DBSession, user: User, chat: Chat, ref: str) -> Doc: + _needs_library(db, user) + skill = skills_service.get(db, ref, user) + if skill is None: + raise Refused("That skill is not there any more.") + return Doc( + key=f"{SOURCE_SKILL}:{skill.id}", + title=skill.name or "Skill", + subtitle="Skill", + text=skill.body or "", + revision=_stamp(skill, skill.body or ""), + writable=sharing.can_write(skill, user), + language="skill.md", + markdown=True, + ) + + +async def _load_doc(db: DBSession, user: User, chat: Chat, ref: str) -> Doc: + _needs_library(db, user) + document = documents_service.get(db, ref, user) + if document is None: + raise Refused("That document is not there any more.") + return Doc( + key=f"{SOURCE_DOC}:{document.id}", + title=document.title or document.filename or "Document", + subtitle="Knowledge document", + text=document.extracted_text or document.extraction_error or "", + revision=_stamp(document, document.extracted_text or ""), + writable=documents_service.can_write(document, user), + language=document.filename or "", + markdown=(document.filename or "").lower().endswith((".md", ".markdown")), + truncated=bool(document.truncated), + ) + + +async def _load_file(db: DBSession, user: User, chat: Chat, ref: str) -> Doc: + attachment = db.get(Attachment, ref) + if attachment is None or attachment.user_id != user.id: + raise Refused("That attachment is not there any more.") + # Belonging to this conversation, so a canvas cannot browse another one's + # files by id. `chat_id` covers one still in the composer; the message check + # covers one that has been sent. + if attachment.chat_id != chat.id: + raise Refused("That attachment belongs to another chat.") + return Doc( + key=f"{SOURCE_FILE}:{attachment.id}", + title=attachment.filename or "Attachment", + subtitle=attachment.source_path or "Attachment", + text=attachment.extracted_text or attachment.extraction_error or "", + # Read-only, and not for want of a write path: `DELETE /api/files/{id}` + # already refuses once the attachment has been sent, because it would + # rewrite a message somebody already read. Editing is the same act with + # a quieter failure. + writable=False, + language=attachment.filename or "", + markdown=(attachment.filename or "").lower().endswith((".md", ".markdown")), + truncated=bool(attachment.truncated), + ) + + +async def _load_scratch(db: DBSession, user: User, chat: Chat, ref: str) -> Doc: + if ref != chat.id: + raise Refused("That scratch document belongs to another chat.") + doc = scratch_service.for_chat(db, chat) + return Doc( + key=f"{SOURCE_SCRATCH}:{chat.id}", + title=doc.title or "Scratch", + subtitle="This chat's scratch document", + text=doc.body or "", + revision=_stamp(doc, doc.body or ""), + writable=True, + language="scratch.md", + markdown=True, + ) + + +# --- Saving -------------------------------------------------------------------------- +async def _save_agent(db: DBSession, user: User, chat: Chat, ref: str, text: str, revision: str): + executor = _executor(db, user, chat) + path = path_key(chat.project_dir, ref) + try: + await executor.write_text(path, text, if_unchanged=revision) + except ExecError as exc: + raise Refused(str(exc)) from exc + + profile = agent_ready(db, user, chat) + if profile is not None: + # Unconditionally, unlike `file_edit` -- whose skip is an optimisation + # for the model's hot path on the grounds that the file was already + # there. The canvas can create one, and a listing known to be wrong is + # what the cache note warns about. + index_service.forget_dir(profile.id, chat.project_dir) + if instructions_service.is_instruction_file(path, chat.project_dir): + instructions_service.forget(profile.id, chat.project_dir) + return await _load_agent(db, user, chat, path) + + +async def _save_note(db: DBSession, user: User, chat: Chat, ref: str, text: str, revision: str): + note = notes_service.get(db, ref, user) + if note is None: + raise Refused("That note is not there any more.") + if not sharing.can_write(note, user): + raise Refused("That note is not yours to change.") + _check_stamp(note, note.body or "", revision) + notes_service.update(db, note, body=text) + return await _load_note(db, user, chat, ref) + + +async def _save_skill(db: DBSession, user: User, chat: Chat, ref: str, text: str, revision: str): + skill = skills_service.get(db, ref, user) + if skill is None: + raise Refused("That skill is not there any more.") + if not sharing.can_write(skill, user): + raise Refused("That skill is not yours to change.") + _check_stamp(skill, skill.body or "", revision) + # Snapshots into a SkillRevision first, which is why a skill needs no + # conflict story beyond the token: a clobber is recoverable. + skills_service.update(db, skill, body=text, note="Edited in the canvas") + return await _load_skill(db, user, chat, ref) + + +async def _save_doc(db: DBSession, user: User, chat: Chat, ref: str, text: str, revision: str): + document = documents_service.get(db, ref, user) + if document is None: + raise Refused("That document is not there any more.") + if not documents_service.can_write(document, user): + raise Refused("That document is not yours to change.") + _check_stamp(document, document.extracted_text or "", revision) + documents_service.set_text(db, document, text) + return await _load_doc(db, user, chat, ref) + + +async def _save_scratch(db: DBSession, user: User, chat: Chat, ref: str, text: str, revision: str): + if ref != chat.id: + raise Refused("That scratch document belongs to another chat.") + doc = scratch_service.for_chat(db, chat) + _check_stamp(doc, doc.body or "", revision) + scratch_service.update(db, doc, body=text) + return await _load_scratch(db, user, chat, ref) + + +# --- One table ------------------------------------------------------------------------- +_SOURCES: dict[str, tuple] = { + SOURCE_AGENT: (_load_agent, _save_agent), + SOURCE_NOTE: (_load_note, _save_note), + SOURCE_SKILL: (_load_skill, _save_skill), + SOURCE_DOC: (_load_doc, _save_doc), + SOURCE_FILE: (_load_file, None), + SOURCE_SCRATCH: (_load_scratch, _save_scratch), +} + + +async def load(db: DBSession, user: User, chat: Chat, key: str) -> Doc: + source, ref = split(key) + entry = _SOURCES.get(source) + if entry is None or not ref: + raise Refused("There is nothing to open here.") + return await entry[0](db, user, chat, ref) + + +async def save( + db: DBSession, user: User, chat: Chat, key: str, text: str, revision: str = "" +) -> Doc: + source, ref = split(key) + entry = _SOURCES.get(source) + if entry is None or not ref: + raise Refused("There is nothing to save here.") + saver = entry[1] + if saver is None: + raise Refused("This one can only be read.") + return await saver(db, user, chat, ref, text, revision) + + +# --- Small shared pieces ------------------------------------------------------------------ +def _needs_library(db: DBSession, user: User) -> None: + if not permissions.has(db, user, "library.use"): + raise Refused("You do not have access to the library.") + + +def _stamp(row, text: str) -> str: + """A revision token for a database row. + + `updated_at` alone would not move for two saves inside one clock tick, so + the length rides along -- the same pairing the file token uses, and for the + same reason. The text is passed in rather than guessed at: a note keeps it + in `body` and a document in `extracted_text`, and a getattr chain that + silently found neither would hand every row the same token. + """ + when = getattr(row, "updated_at", None) + if when is not None and when.tzinfo is None: + # SQLite does not store the offset, so a row loaded from disk comes back + # naive while one still in the session's identity map keeps the tzinfo + # it was created with -- and `.timestamp()` reads a naive value as local + # time. Without this the same row yields two different tokens depending + # on where it was loaded, and every save outside UTC would report a + # conflict that is not there. The same normalisation + # `compaction.moment` makes, for the same reason. + when = when.replace(tzinfo=UTC) + return revision_of(int(when.timestamp()) if when else 0, len(text or "")) + + +def _check_stamp(row, text: str, revision: str) -> None: + """Refuse a save whose token no longer matches. An empty token overwrites. + + Empty is what Overwrite on the conflict card sends: somebody has been shown + both versions and chosen. Never save silently over a change; never discard + silently either. + """ + if revision and _stamp(row, text) != revision: + raise Conflict(_stamp(row, text)) diff --git a/src/lembas/services/generation.py b/src/lembas/services/generation.py index e71da32..30bb954 100644 --- a/src/lembas/services/generation.py +++ b/src/lembas/services/generation.py @@ -28,6 +28,7 @@ from sqlalchemy import select from lembas.db.models import KIND_AGENT, ROLE_ASSISTANT, ROLE_USER, Chat, Message, User from lembas.db.session import session_scope +from lembas.services import canvas as canvas_service from lembas.services import chat as chat_service from lembas.services import compaction as compaction_service from lembas.services import interaction, settings_store, tokens, tool_labels @@ -167,6 +168,12 @@ class Generation: # `_persist` stays one writer with one rule; only this decides whether the # tools are withdrawn for a final round. plan_final: bool = False + # Which files this reply has put in the canvas panel. Seeded once from + # `chat.canvas_json` where `_run` already has the chat loaded, then mutated + # in place -- two `file_read` calls in one round that each re-read the row + # would leave only the second, which is the lost update `plan` above + # documents. Folded back by `_persist`, the single writer. + canvas: dict = field(default_factory=dict) # The queue, seen from the reply's side. `drained` says this reply's ending # handed the next waiting prompt to a fresh one; `injected_ids` names the # prompts taken into *this* reply between two rounds of tool calls. Both are @@ -413,6 +420,12 @@ async def _run(generation: Generation) -> None: # worth asking for. Read here with the rest, because titling happens # after this session has closed. title_from_prompt = chat.kind == KIND_AGENT + # Seeded once, here, where the chat is already loaded. Mutated from + # then on; see the field's own note. + generation.canvas = { + "tabs": list((chat.canvas_json or {}).get("tabs") or []), + "active": (chat.canvas_json or {}).get("active") or "", + } # Read here, with the rest, because titling happens after this # session has closed and must not open another one. title_prompt = prompts_service.resolve(db, "task.title") @@ -643,6 +656,13 @@ async def _run(generation: Generation) -> None: generation.tool_events.append(outcome.event) generation.output_bytes += len(outcome.content) messages.append(tools_service.tool_turn(call, outcome.content)) + if opened := outcome.event.get("canvas"): + # A runner cannot write the message row, so the loop carries + # this exactly as it carries a merged plan. Never activated: + # an agent reads forty files in a long reply, and dragging + # somebody through all of them -- or away from a file they + # are editing -- is what makes a panel like this unusable. + canvas_service.open_tab(generation.canvas, opened, activate=False) if outcome.event.get("plan"): generation.plan = outcome.event["plan"] # Only `plan_submit` sets this. `plan_update` writes the @@ -1591,6 +1611,11 @@ def _persist(generation: Generation, title: str, elapsed: float) -> None: message.reasoning_ms = generation.reasoning_ms message.tool_calls_json = generation.tool_events message.plan_json = generation.plan or {} + if generation.canvas.get("tabs"): + # A union with whatever the row says *now*, not an overwrite: + # the snapshot above was seeded when the reply began, and + # somebody may have opened a tab by hand since. + chat.canvas_json = canvas_service.merge(chat.canvas_json, generation.canvas) if generation.plan: # This bubble now carries the plan in force, and the chat points # at it so the harness can find it with one primary-key lookup diff --git a/src/lembas/services/library/documents.py b/src/lembas/services/library/documents.py index 8b2f49c..12ef76d 100644 --- a/src/lembas/services/library/documents.py +++ b/src/lembas/services/library/documents.py @@ -254,6 +254,46 @@ def get(db: DBSession, document_id: str, user: User | None) -> Document | None: return document +def can_write(document: Document, user: User | None) -> bool: + """Whether this person may change a document's text. + + Ownership, through the same helper every other library store uses. Sharing + grants **reading only**, so being able to see a document through somebody + else's base is never enough to rewrite it -- and reading is already settled + by `get`, which resolves visibility through the base. + + Its own function rather than `sharing.can_write` at the call site because + `Document` is the one store whose visibility does not come from itself, and + a reader arriving at a bare `sharing.can_write(document, …)` would have to + go and check whether that is the right question. + """ + return sharing.can_write(document, user) + + +def set_text(db: DBSession, document: Document, text: str) -> Document: + """Replace the extracted text a person reads and a model searches. + + The stored file is untouched: the bytes are the record, and this is what was + made of them. That is the same line PDF extraction draws -- extracted once + at upload, so a reply cannot change because a parser was upgraded -- and it + is why editing this is safe for transcripts: `files.copy_document` copies + the text when a document is attached, so an edit only changes what future + searches find. + + `extraction_error` is cleared, because replacing a failed extraction by hand + is the main reason to want this at all; leaving the old apology beside the + new text would be the page contradicting itself. + + The commit fires the `documents_fts` UPDATE trigger, so search stays correct + with nothing else to do. See `db/migrations.py:ensure_fts`. + """ + document.extracted_text = text[:files_service.MAX_EXTRACTED_CHARS] + document.truncated = len(text) > files_service.MAX_EXTRACTED_CHARS + document.extraction_error = "" + db.commit() + return document + + def search( db: DBSession, user: User | None, diff --git a/src/lembas/services/markdown.py b/src/lembas/services/markdown.py index 4a561ba..0afa338 100644 --- a/src/lembas/services/markdown.py +++ b/src/lembas/services/markdown.py @@ -19,7 +19,7 @@ import nh3 from markdown_it import MarkdownIt from pygments import highlight from pygments.formatters import HtmlFormatter -from pygments.lexers import get_lexer_by_name, guess_lexer +from pygments.lexers import get_lexer_by_name, get_lexer_for_filename, guess_lexer from pygments.util import ClassNotFound # Class-based highlighting; the colours come from theme tokens in chat.css, so @@ -95,6 +95,43 @@ def _render_fence(tokens, idx, _options, _env) -> str: ) +def highlight_code(text: str, filename: str = "") -> str: + """A whole file, class-highlighted, for the canvas panel to read. + + Here rather than in a module of its own because `markdown.py` is where + pygments lives and `_FORMATTER` is already configured: a second formatter + would mean a second set of class names and a second thing to theme, and the + `.pg-*` rules would then be right about code fences and wrong about files. + + Pygments' `HtmlFormatter` escapes what it is given, which is what makes this + the one call the canvas templates mark `|safe`. The content came off + somebody else's disk, so that property is the whole of the argument -- if + the lexer cannot be found the text is escaped by hand instead, never passed + through. + + Chooses by filename, because that is what the canvas has: a lexer guessed + from contents is confidently wrong on short files, and there is no fence + info string here to read a language out of. + """ + if not text: + return "" + + lexer = None + if filename: + try: + lexer = get_lexer_for_filename(filename, stripall=False) + except (ClassNotFound, ValueError): + lexer = None + if lexer is None and len(text) > 200: + try: + lexer = guess_lexer(text) + except (ClassNotFound, ValueError): + lexer = None + + body = nh3.clean_text(text) if lexer is None else highlight(text, lexer, _FORMATTER) + return f'
{body}
' + + @functools.lru_cache(maxsize=1) def _parser() -> MarkdownIt: md = MarkdownIt("commonmark", {"linkify": True, "typographer": False}) diff --git a/src/lembas/services/prompts.py b/src/lembas/services/prompts.py index 0330635..e05b5af 100644 --- a/src/lembas/services/prompts.py +++ b/src/lembas/services/prompts.py @@ -924,6 +924,27 @@ BUILTIN: tuple[Fragment, ...] = ( "since that is all you will see next time." ), ), + Fragment( + key="tool.scratch", + label="The scratch document", + group=GROUP_TOOLS, + order=245, + families=("scratch",), + hint="Appears when scratch_write is offered. The point worth making to " + "a model is the one it cannot infer from the schema: this is *watched* " + "while it is written, so building something up here is visible work " + "rather than a result announced at the end — and it is not searchable " + "afterwards, which is what keeps it from being used as a note.", + default=( + "- This chat has a scratch document, open beside the conversation and visible " + "to the person as you write it. Use scratch_write for something you build up " + "as you work — a draft, a table of findings, a list you keep adding to — " + "rather than repeating the whole thing in each reply. Append unless you mean " + "to start again. They can edit it themselves and attach it to a later message. " + "It belongs to this chat and cannot be searched afterwards, so anything worth " + "keeping beyond it is a note." + ), + ), # --- Context ------------------------------------------------------------- Fragment( key="context.knowledge_scope", diff --git a/src/lembas/services/scratch.py b/src/lembas/services/scratch.py new file mode 100644 index 0000000..4361e81 --- /dev/null +++ b/src/lembas/services/scratch.py @@ -0,0 +1,82 @@ +"""A chat's own working surface. + +One text artefact per chat, written by the model through `scratch_write` and by +the person through the canvas panel, and handed to a message as an ordinary +attachment when it is ready. + +Not a note. A note is a durable artefact of the reader's that outlives the chat +and is searchable; this is the chat's own record of what it is working on, which +is the line `plan_update` sits on rather than `notes_edit`. It is deliberately +not injected as context every turn either -- that is what a memory is for, and a +working document injected whole on every request is how a window fills up. +""" + +from __future__ import annotations + +from sqlalchemy import select +from sqlalchemy.orm import Session as DBSession + +from lembas.db.models import AUTHOR_USER, Chat, ScratchDoc + +# Between a note's 40k and an attachment's 120k. Large enough to hold a draft +# somebody is actually working on, small enough that attaching one does not +# quietly cost most of a context window. +MAX_BODY_CHARS = 100_000 +MAX_TITLE_CHARS = 300 + + +def get(db: DBSession, chat: Chat) -> ScratchDoc | None: + """The chat's pad, or None. No side effect. + + Separate from `for_chat` because the harness and the mention picker ask + whether there is one, and a question must not create the thing it asks + about -- otherwise every chat ever opened acquires an empty row. + """ + return db.scalar(select(ScratchDoc).where(ScratchDoc.chat_id == chat.id)) + + +def for_chat(db: DBSession, chat: Chat) -> ScratchDoc: + """The chat's pad, made if it is not there yet.""" + existing = get(db, chat) + if existing is not None: + return existing + doc = ScratchDoc(chat_id=chat.id, user_id=chat.user_id) + db.add(doc) + db.commit() + return doc + + +def update( + db: DBSession, + doc: ScratchDoc, + *, + body: str | None = None, + title: str | None = None, + author: str = AUTHOR_USER, +) -> ScratchDoc: + """Replace what is in the pad. Absent arguments are left alone. + + The body is **not** stripped, unlike a note's. This is a document somebody + is editing, and trailing whitespace they typed is theirs -- a save that + silently trims the line you are standing on is the kind of thing that makes + an editor feel broken. + """ + if title is not None and title.strip(): + doc.title = title.strip()[:MAX_TITLE_CHARS] + if body is not None: + doc.body = body[:MAX_BODY_CHARS] + doc.author = author + db.commit() + return doc + + +def append(db: DBSession, doc: ScratchDoc, text: str, *, author: str) -> ScratchDoc: + """Add to the end, with a blank line between what was there and what is new. + + Its own function rather than the caller reading and concatenating, because + two calls in one round would otherwise each read the same body and the + second would drop the first -- the same lost update `plan_update` documents. + """ + existing = doc.body or "" + joined = f"{existing.rstrip()}\n\n{text}" if existing.strip() else text + return update(db, doc, body=joined, author=author) diff --git a/src/lembas/services/tool_labels.py b/src/lembas/services/tool_labels.py index eb5d4e1..c20f461 100644 --- a/src/lembas/services/tool_labels.py +++ b/src/lembas/services/tool_labels.py @@ -61,6 +61,7 @@ LABELS: dict[str, str] = { "notes_create": "Note written", "notes_edit": "Note updated", "notes_delete": "Note deleted", + "scratch_write": "Canvas written", "memory_add": "Memory saved", "memory_forget": "Memory removed", "skill_get": "Skill read", @@ -92,6 +93,7 @@ ICONS: dict[str, str] = { "notes_create": "pencil", "notes_edit": "pencil", "notes_delete": "trash", + "scratch_write": "file-text", "memory_add": "star", "memory_forget": "trash", "skill_get": "sparkle", @@ -127,6 +129,7 @@ ACTIONS: dict[str, str] = { "notes_create": "Write a note", "notes_edit": "Change a note", "notes_delete": "Delete a note", + "scratch_write": "Write in the canvas", "memory_add": "Remember something", "memory_forget": "Forget something", "skill_get": "Read a skill", diff --git a/src/lembas/services/tools.py b/src/lembas/services/tools.py index 51a8baf..3fb9962 100644 --- a/src/lembas/services/tools.py +++ b/src/lembas/services/tools.py @@ -35,6 +35,7 @@ from sqlalchemy.orm import Session as DBSession from lembas.db.models import AUTHOR_MODEL, Chat, User from lembas.db.session import session_scope from lembas.services import prompts as prompts_service +from lembas.services import scratch as scratch_service from lembas.services import search as search_service from lembas.services import settings_store from lembas.services.library import documents as documents_service @@ -94,6 +95,13 @@ FAMILY_MCP = "mcp" # is the only tool the model cannot resolve by itself. FAMILY_ASK = "ask" +# The chat's own working surface -- the canvas panel's scratch document. +# Deliberately not part of `notes`: a note is a durable artefact of the reader's +# that outlives the chat and is searchable, while this is the chat's own record +# of what it is doing, which is the line `plan_update` sits on. It is also its +# own switch, because narrowing notes off must not silently take the pad too. +FAMILY_SCRATCH = "scratch" + # Acting on the machine an agent chat is pointed at. Offered only when the chat # is one, has a usable connection, and the feature is switched on -- see # services/agent/session.py:resolve, which answers all three at once. @@ -107,6 +115,7 @@ FAMILIES = ( FAMILY_NOTES, FAMILY_MEMORY, FAMILY_SKILLS, + FAMILY_SCRATCH, FAMILY_ASK, FAMILY_AGENT, ) @@ -507,6 +516,51 @@ async def _run_notes_delete(context: ToolContext, args: dict[str, Any]) -> ToolO ) +# --- The chat's scratch document --------------------------------------------- +async def _run_scratch_write(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + """Write into the pad the person can see beside the conversation. + + Opens its own session, like every other runner: a generation outlives the + session that resolved it. + + `append` is a service function rather than a read-and-concatenate here, + because two calls in one round would otherwise each read the same body and + the second would drop the first. + """ + with session_scope() as db: + chat = db.get(Chat, context.chat_id) if context.chat_id else None + if chat is None: + return ToolOutcome( + "There is no chat to write into.", + {"name": "scratch_write", "status": "error", "error": "No chat."}, + ) + doc = scratch_service.for_chat(db, chat) + text = str(args.get("text") or "") + if str(args.get("mode") or "append").strip().lower() == "replace": + scratch_service.update(db, doc, body=text, author=AUTHOR_MODEL) + what = "Replaced" + else: + scratch_service.append(db, doc, text, author=AUTHOR_MODEL) + what = "Added to" + return ToolOutcome( + f"{what} the scratch document ({len(doc.body)} characters). " + "It is on screen beside the conversation.", + { + "name": "scratch_write", + "query": doc.title, + "status": "ok", + "results": [], + # Opens the tab, the same way a file tool does. Never brings it + # to the front -- see `canvas.open_tab`. + "canvas": { + "key": f"scratch:{chat.id}", + "title": doc.title, + "source": "scratch", + }, + }, + ) + + # --- Memory ------------------------------------------------------------------ async def _run_memory_add(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: content = str(args.get("content") or "").strip() @@ -819,6 +873,38 @@ REGISTRY: dict[str, ToolDef] = { run=_run_notes_delete, risk=RISK_WRITE, ), + ToolDef( + name="scratch_write", + family=FAMILY_SCRATCH, + description=( + "Write into this chat's scratch document, which the person can " + "see and edit beside the conversation. Use it for something you " + "are building up as you work — a draft, a table of findings, a " + "list you keep adding to — rather than putting it in the reply " + "and rewriting the whole thing each turn. It is not searchable " + "later and belongs to this chat alone; use a note for anything " + "worth keeping beyond it." + ), + parameters=_object( + { + "mode": { + **_STRING, + "enum": ["append", "replace"], + "description": "append is the default.", + }, + "text": {**_STRING, "description": "Markdown."}, + }, + ["text"], + ), + run=_run_scratch_write, + # What a tool does to the *world the four modes govern*, which is the + # machine -- and this cannot touch it. RISK_WRITE would put an + # approval card on screen every time the model jotted a paragraph, + # which is exactly the interruption batching exists to prevent. The + # same argument `plan_update` carries. An administrator who + # disagrees puts it in `deny_default`. + risk=RISK_READ, + ), ToolDef( name="memory_add", family=FAMILY_MEMORY, @@ -990,11 +1076,12 @@ def _family_allowed( # attach path keeps working, because that one is a person's instruction # rather than a model's choice. return bool(allowed.get("tools.fetch") and config.get("fetch_enabled")) - if gate in (FAMILY_CUSTOM, FAMILY_MCP, FAMILY_ASK, FAMILY_AGENT): + if gate in (FAMILY_CUSTOM, FAMILY_MCP, FAMILY_ASK, FAMILY_AGENT, FAMILY_SCRATCH): # Deliberately without `library.use`: an HTTP endpoint an administrator # wrote has nothing to do with this person's own documents and notes, # and requiring the library permission for it would be a coincidence of - # naming rather than a rule. The same goes for being asked a question. + # naming rather than a rule. The same goes for being asked a question, + # and for a pad that belongs to this chat and goes nowhere else. return bool(allowed.get(f"tools.{gate}")) return bool(allowed.get(f"tools.{gate}") and allowed.get("library.use")) diff --git a/src/lembas/web/static/css/app.css b/src/lembas/web/static/css/app.css index 7268f22..dd6c5c3 100644 --- a/src/lembas/web/static/css/app.css +++ b/src/lembas/web/static/css/app.css @@ -588,7 +588,8 @@ body.is-resizing { cursor: col-resize; user-select: none; } -body.is-resizing .terminal__screen { pointer-events: none; } +body.is-resizing .terminal__screen, +body.is-resizing .canvas__body { pointer-events: none; } @media (max-width: 64rem) { /* A full-height overlay has no edge to drag, and no room to spare. */ @@ -644,14 +645,179 @@ body.is-resizing .terminal__screen { pointer-events: none; } .terminal__last:empty { display: none; } .terminal__message--error { color: var(--danger); } +/* --- The canvas panel ------------------------------------------------------ + The same shape as the terminal beside it: a fixed-width column that hides + with the `hidden` attribute, and shares .panel-head and .panel-resize. It + sits nearest the conversation, being the widest and the one most likely to + be read alongside it. */ +.canvas { + width: var(--canvas-width); + min-width: var(--canvas-width-min); + max-width: 80vw; + flex: none; + display: flex; + flex-direction: column; + min-height: 0; + position: relative; + background: var(--bg-sunken); + border-left: 1px solid var(--border); +} +.canvas__inner { + display: flex; + flex-direction: column; + min-height: 0; + flex: 1; +} +/* Where the file came from, beside its name. Shrinks and truncates rather than + pushing the close button off the end. */ +.canvas__where { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--ink-faint); + font-family: var(--font-mono); + font-size: var(--text-xs); + font-weight: 400; +} + +/* One row, always. It scrolls sideways rather than wrapping -- the same rule + the composer's toolbar is built around, and for the same reason: a strip + that wraps to three lines takes the file with it. */ +.canvas__tabs { + display: flex; + flex: none; + gap: var(--sp-1); + padding: var(--sp-1) var(--sp-2); + overflow-x: auto; + scrollbar-width: thin; + border-bottom: 1px solid var(--border); +} +.canvas__tab { + display: inline-flex; + align-items: center; + flex: none; + max-width: 14rem; + border-radius: var(--radius-sm); + background: transparent; + transition: background var(--transition-fast); +} +.canvas__tab:hover { background: var(--surface); } +.canvas__tab.is-active { background: var(--surface-raised); } +.canvas__tab-open { + display: inline-flex; + align-items: center; + gap: var(--sp-1); + min-width: 0; + height: var(--control-h-sm); + padding: 0 var(--sp-1) 0 var(--sp-2); + border: 0; + background: none; + color: var(--ink-muted); + font-size: var(--text-xs); + cursor: pointer; +} +.canvas__tab.is-active .canvas__tab-open { color: var(--ink); } +.canvas__tab-label { + min-width: 0; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} +/* Unsaved. A dot rather than a colour alone, which nobody can see in a theme + they did not choose. */ +.canvas__tab-dot { + flex: none; + width: 6px; + height: 6px; + border-radius: var(--radius-full); + background: var(--accent); +} +.canvas__tab-close { + display: inline-flex; + align-items: center; + padding: 0 var(--sp-1); + border: 0; + background: none; + color: var(--ink-faint); + cursor: pointer; +} +.canvas__tab-close:hover { color: var(--ink); } + +/* One row, and the path box is the only thing allowed to shrink -- the same + arrangement the composer's toolbar is built around. */ +.canvas__open { + display: flex; + align-items: center; + gap: var(--sp-2); + flex: none; + padding: var(--sp-2) var(--sp-3); + border-bottom: 1px solid var(--border); +} +.canvas__open-form { display: flex; gap: var(--sp-2); min-width: 0; flex: 1; } +.canvas__path { + min-width: 0; + flex: 1; + height: var(--control-h-sm); + font-size: var(--text-xs); +} + +.canvas__body { + flex: 1; + min-height: 0; + overflow: auto; + padding: var(--sp-3); +} +.canvas__doc { display: flex; flex-direction: column; gap: var(--sp-2); } +.canvas__actions { display: flex; align-items: center; gap: var(--sp-2); } +.canvas__hint, .canvas__empty, .canvas__note { + color: var(--ink-faint); + font-size: var(--text-xs); +} +.canvas__note { display: flex; align-items: center; gap: var(--sp-2); } +.canvas__code { + margin: 0; + padding: var(--sp-3); + border-radius: var(--radius-sm); + background: var(--code-bg); + font-family: var(--font-mono); + font-size: var(--text-xs); + line-height: 1.6; + white-space: pre; + overflow-x: auto; +} +.canvas__form { display: flex; flex-direction: column; gap: var(--sp-2); } +/* No highlighting while typing, and the panel says so rather than pretending. + A mirror behind this would be the composer's trick at two thousand lines, + laying the buffer out twice on every keystroke. */ +.canvas__editor { + width: 100%; + min-height: 24rem; + padding: var(--sp-3); + border: 1px solid var(--border); + border-radius: var(--radius-sm); + background: var(--code-bg); + color: var(--ink); + font-family: var(--font-mono); + font-size: var(--text-xs); + line-height: 1.6; + white-space: pre; + overflow-wrap: normal; + resize: vertical; +} +.canvas__conflict { display: flex; flex-direction: column; gap: var(--sp-3); } +.canvas__theirs summary { cursor: pointer; font-size: var(--text-xs); } + @media (max-width: 64rem) { - .terminal { + .terminal, + .canvas { position: fixed; inset: 0 0 0 auto; - width: min(var(--terminal-width), 100vw); z-index: var(--z-panel); box-shadow: var(--shadow-lg); } + .terminal { width: min(var(--terminal-width), 100vw); } + .canvas { width: min(var(--canvas-width), 100vw); } } .topbar { diff --git a/src/lembas/web/static/css/tokens.css b/src/lembas/web/static/css/tokens.css index 50ccadf..13790d4 100644 --- a/src/lembas/web/static/css/tokens.css +++ b/src/lembas/web/static/css/tokens.css @@ -72,6 +72,13 @@ re-wraps everything a program prints. */ --terminal-width: 34rem; --terminal-width-min: 24rem; + /* Wider again: a source line is longer than eighty columns once nothing is + re-wrapping it, and this one holds prose as well. The minimum is 24rem = + 384px and must equal both `data-resize-min` in chat/_canvas.html and the + lower bound in api/preferences.py:LAYOUT_BOUNDS -- a width outside those + bounds is silently dropped, so the three are pinned equal by a test. */ + --canvas-width: 40rem; + --canvas-width-min: 24rem; --thread-max-width: 48rem; --header-height: 3.5rem; diff --git a/src/lembas/web/static/js/canvas.js b/src/lembas/web/static/js/canvas.js new file mode 100644 index 0000000..18a4496 --- /dev/null +++ b/src/lembas/web/static/js/canvas.js @@ -0,0 +1,111 @@ +/* + The canvas panel's small amount of behaviour. + + Almost all of it is htmx: the tabs post, the save posts, the panel swaps. + Three things need JavaScript, and only three. + + 1. Tab inserts a tab character instead of leaving the field. Without it the + one editor affordance whose absence is genuinely maddening is missing. + 2. The editor tracks whether it has been changed, so the tab shows a dot and + so leaving the page with unsaved work warns. + 3. Opening the panel scrolls the active tab into view, since the strip + scrolls sideways and the tab in front may be off the end of it. + + What is *not* here is any guard against a swap taking the editor away. A + model opening a file sends the tab strip and nothing else, and does not move + the active tab -- both settled on the server, where they cannot be lost to a + race. + + Nothing here ever assigns innerHTML from a fetch: every swap is htmx's, and + the content is a file off somebody else's disk. +*/ +(function () { + "use strict"; + + var panel = null; + /* Whether the editor has been changed since it was last rendered. Module + state rather than a data attribute, because the element it describes is + replaced by every swap and an attribute would go with it. */ + var dirty = false; + + function markDirty(on) { + dirty = !!on; + var dot = panel && panel.querySelector(".canvas__tab.is-active [data-canvas-dirty]"); + if (dot) dot.hidden = !dirty; + } + + /* --- Typing ------------------------------------------------------------- */ + function onInput(event) { + if (event.target && event.target.matches("[data-canvas-editor]")) markDirty(true); + } + + function onKeydown(event) { + var box = event.target; + if (!box || !box.matches || !box.matches("[data-canvas-editor]")) return; + if (event.key !== "Tab" || event.ctrlKey || event.altKey || event.metaKey) return; + /* Shift+Tab still leaves the field, which is the only way out of it for + somebody using the keyboard. */ + if (event.shiftKey) return; + + event.preventDefault(); + var start = box.selectionStart; + var end = box.selectionEnd; + box.value = box.value.slice(0, start) + "\t" + box.value.slice(end); + box.selectionStart = box.selectionEnd = start + 1; + markDirty(true); + } + + /* --- Swaps -------------------------------------------------------------- */ + /* There is deliberately nothing here guarding the editor against a swap. + A model opening a file sends the tab *strip* and nothing else -- the body + is never pushed -- and `canvas.open_tab` does not move the active tab for + a model, so the file in front and the field being typed in both stay put. + That is settled on the server, where it cannot be lost to a race. */ + function onAfterSwap(event) { + if (!panel || !event.target || !panel.contains(event.target)) return; + /* The server has just rendered what is stored, so nothing is unsaved until + somebody types again. */ + markDirty(false); + showActiveTab(); + } + + function showActiveTab() { + var active = panel && panel.querySelector(".canvas__tab.is-active"); + if (active && active.scrollIntoView) { + active.scrollIntoView({ block: "nearest", inline: "nearest" }); + } + } + + /* --- Leaving with unsaved work ------------------------------------------ */ + function onBeforeUnload(event) { + if (!dirty) return; + event.preventDefault(); + /* The browser shows its own wording; returning a string is what makes older + ones show anything at all. */ + event.returnValue = ""; + return ""; + } + + /* --- Wiring ------------------------------------------------------------- */ + function start() { + panel = document.querySelector("[data-canvas]"); + if (!panel) return; + + panel.addEventListener("input", onInput); + panel.addEventListener("keydown", onKeydown); + panel.addEventListener("lembas:toggle", function (event) { + if (event.detail && event.detail.open) showActiveTab(); + }); + + /* On document, not on the panel: htmx fires these on the element being + swapped, and by the time afterSwap runs the old node is gone. */ + document.body.addEventListener("htmx:afterSwap", onAfterSwap); + window.addEventListener("beforeunload", onBeforeUnload); + } + + if (document.readyState === "loading") { + document.addEventListener("DOMContentLoaded", start); + } else { + start(); + } +})(); diff --git a/src/lembas/web/static/js/commands.js b/src/lembas/web/static/js/commands.js index f127dd3..e76720b 100644 --- a/src/lembas/web/static/js/commands.js +++ b/src/lembas/web/static/js/commands.js @@ -41,6 +41,7 @@ { keys: "Alt + M", what: "Dictate" }, { keys: "Alt + R", what: "Read the last reply aloud" }, { keys: "Alt + 1 … 4", what: "Manual, Edit, Auto, Plan" }, + { keys: "Alt + E", what: "Canvas" }, { keys: "Alt + T", what: "Terminal" }, { keys: "Alt + I", what: "Inspector" }, { keys: "Alt + B", what: "Sidebar" }, @@ -135,6 +136,15 @@ when: function () { return !!chat() && isAgent(); }, run: function () { reindex(); } }, + { + name: "canvas", + summary: "Show or hide the canvas", + /* Gated, like the other two panels. An ungated command on a page with no + panel does not merely fail -- it stops being a command, and the message + is sent as written. */ + when: function () { return !!el("#canvas"); }, + run: function () { toggle("#canvas", "side"); } + }, { name: "terminal", summary: "Show or hide the terminal", @@ -526,6 +536,13 @@ return; } + /* E for editor, not C: Ctrl/Cmd+C is too near for comfort, and Alt+D is + the address bar in two browsers -- a shortcut the browser wins looks + broken. */ + if (event.code === "KeyE" && el("#canvas")) { + event.preventDefault(); + return toggle("#canvas", "side"); + } if (event.code === "KeyT" && el("#terminal")) { event.preventDefault(); return toggle("#terminal", "side"); diff --git a/src/lembas/web/templates/chat/_canvas.html b/src/lembas/web/templates/chat/_canvas.html new file mode 100644 index 0000000..350c08b --- /dev/null +++ b/src/lembas/web/templates/chat/_canvas.html @@ -0,0 +1,48 @@ +{% from "_macros.html" import icon %} +{# + The canvas panel: files, open beside the conversation. + + Built the way the terminal panel is -- a child of .shell, `hidden` until a + toggle removes it, its own drag handle, the shared panel head. Filled the way + the *inspector* is, though: `hx-trigger="intersect once"`, because a hidden + element never intersects, so a panel nobody opens costs one element and no + round trip to somebody's machine. There is nothing heavy to construct here, + which is the whole reason it does not need the terminal's lazy-build dance. + + Everything shown inside came off somebody else's disk, or out of a model. It + is rendered through pygments (which escapes), through render_markdown (the one + path allowed to emit HTML), or into a +
+ + +
+ + diff --git a/src/lembas/web/templates/chat/_canvas_doc.html b/src/lembas/web/templates/chat/_canvas_doc.html new file mode 100644 index 0000000..f23d3dd --- /dev/null +++ b/src/lembas/web/templates/chat/_canvas_doc.html @@ -0,0 +1,74 @@ +{% from "_macros.html" import icon %} +{# + One file, read or edited. + + A read/edit split rather than a highlighting editor, because there is no + vendored code editor and adding one would be a build step (hard rule 1) or a + payload larger than xterm's on every page in the application -- and xterm is + called out as the one heavy dependency precisely because it loads only on a + chat that can open a terminal. + + So: pygments server-side for reading, and a plain +
+ + + No colour while you type. Tab inserts a tab. +
+ + {% endif %} + diff --git a/src/lembas/web/templates/chat/_canvas_inner.html b/src/lembas/web/templates/chat/_canvas_inner.html new file mode 100644 index 0000000..aa37799 --- /dev/null +++ b/src/lembas/web/templates/chat/_canvas_inner.html @@ -0,0 +1,87 @@ +{% from "_macros.html" import icon %} +{# + The head, the tab strip and whichever file is in front. Everything the panel + swaps, in one fragment. + + Both together, always: rendering only the body would leave the strip showing a + tab that is no longer there after a close, and rendering only the strip would + leave the previous file on screen after a switch. +#} +
+

+ {{ icon("file-text", "icon--sm") }} + {{ doc.title if doc else "Canvas" }} + {% if doc and doc.subtitle %} + {{ doc.subtitle }} + {% endif %} +

+ + {% if doc and doc.key.startswith("scratch:") %} + {# A copy, like every other attach path -- a transcript must not change + because the pad was edited afterwards. #} + + {% endif %} + + +
+ +{% include "chat/_canvas_tabs.html" %} + +{# + Opening one by hand. A path box rather than a file browser: the model opens + what it touches, which is the path this feature is really for, and a second + directory browser beside the one the composer already has would be a lot of + interface for the rarer case. A relative path resolves against the project + directory, exactly as it does for the model. +#} +
+ {% if canvas_agent %} + {# Its own form. A second control named `key` in the same one -- the Scratch + button below -- would send two values for one field, and which of them the + server took would be an accident. #} +
+ + +
+ {% endif %} + + +
+ +
+ {% if error %} + + {% elif conflict %} + {% include "chat/_canvas_conflict.html" %} + {% elif doc %} + {% include "chat/_canvas_doc.html" %} + {% else %} +

+ Nothing open. A file the model reads or writes appears here, and + {% if canvas_agent %}the @ menu can put one here too.{% else %}notes, + skills and this chat's own scratch document can be opened from the @ menu. + {% endif %} +

+ {% endif %} +
diff --git a/src/lembas/web/templates/chat/_canvas_tabs.html b/src/lembas/web/templates/chat/_canvas_tabs.html new file mode 100644 index 0000000..ca7a3f9 --- /dev/null +++ b/src/lembas/web/templates/chat/_canvas_tabs.html @@ -0,0 +1,38 @@ +{% from "_macros.html" import icon %} +{# + One row, always, and it scrolls sideways rather than wrapping -- the same rule + the composer's toolbar is built around. A strip that wraps to three lines on a + narrow panel takes the file with it. + + A tab is a button posting to a route that serves POST. Not a link: `GET` never + moves the active tab, because there is no CSRF token here and the cookie is + SameSite Lax, so a state-changing GET is a link somebody can be made to follow. +#} +{# `oob` is set only when this arrives on the reply's SSE stream, where it has + to find its own way to the panel rather than being swapped into the bubble + the stream is writing. Out of band, exactly as the `done` frame's title is. #} +
+ {% for tab in tabs %} +
+ + +
+ {% endfor %} +
diff --git a/src/lembas/web/templates/chat/_message.html b/src/lembas/web/templates/chat/_message.html index 2bf2227..f6210b9 100644 --- a/src/lembas/web/templates/chat/_message.html +++ b/src/lembas/web/templates/chat/_message.html @@ -284,5 +284,17 @@ {% if streaming %} {# Receives the finished bubble and replaces this whole article with it. #} + + {# + A file the model has opened. The frame carries the canvas tab strip marked + `hx-swap-oob`, so it lands in the panel rather than here -- this element is + only somewhere for it to arrive. `hx-swap="none"` because the payload has no + business in the bubble; htmx extracts out-of-band fragments before it + considers the main swap, so "none" does not stop them. + + Only the strip is ever pushed. The file's contents would be a lot of bytes + on every version bump and would overwrite a textarea somebody is typing in. + #} + {% endif %} diff --git a/src/lembas/web/templates/chat/index.html b/src/lembas/web/templates/chat/index.html index 40ad48f..2480edb 100644 --- a/src/lembas/web/templates/chat/index.html +++ b/src/lembas/web/templates/chat/index.html @@ -89,6 +89,18 @@ {% endif %} {% endif %} + {% if canvas_enabled %} + {# Nearest the conversation of the three, being the widest and the one + most likely to be open beside it. All three share one slot: at + 1280px the sidebar plus two panels leaves about seventy pixels of + chat. #} + + {% endif %} + {% if terminal_enabled %} {# To the left of the inspector, and never open beside it: see the toggle group in app.js. #} @@ -316,8 +328,12 @@ {% endif %} - {# Third and fourth children of .shell, mirroring the sidebar opposite. The - terminal comes first so it sits to the left of the inspector. #} + {# The panels, mirroring the sidebar opposite, in the order they sit on + screen: the canvas nearest the conversation, then the terminal, then the + inspector. Only ever one of them is open -- see the toggle group. #} + {% if canvas_enabled %} + {% include "chat/_canvas.html" %} + {% endif %} {% if terminal_enabled %} {% include "chat/_terminal.html" %} {% endif %} @@ -328,6 +344,9 @@ {% endblock %} {% block scripts %} +{% if canvas_enabled %} + +{% endif %} {% if terminal_enabled %} {# Only where it can be used. xterm is nearly three times everything else vendored, so a plain chat must never load it. #} diff --git a/tests/test_canvas.py b/tests/test_canvas.py new file mode 100644 index 0000000..6ed215c --- /dev/null +++ b/tests/test_canvas.py @@ -0,0 +1,531 @@ +"""The canvas panel: tabs, sources, saving, and what a model may move.""" + +from __future__ import annotations + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import select + +from lembas.db.models import ( + KIND_AGENT, + Attachment, + Chat, + Connection, + Model, + ScratchDoc, + User, +) +from lembas.services import canvas as canvas_service +from lembas.services import scratch as scratch_service +from lembas.services import settings_store +from lembas.services.crypto import encrypt +from lembas.services.library import documents as documents_service +from lembas.services.library import notes as notes_service + + +def _page(db, user, base, text: str): + """A knowledge document, without going near the network.""" + from lembas.services.fetch import Fetched + + return documents_service.store_page( + db, + owner=user, + base=base, + page=Fetched(url="http://example.test/terms", title="Terms", text=text), + ) + + +def _add_connection(db) -> Connection: + connection = Connection( + name="Test", base_url="http://127.0.0.1:1", api_key_encrypted=encrypt("") + ) + db.add(connection) + db.commit() + db.add(Model(connection_id=connection.id, model_id="test-model")) + db.commit() + return connection + + +# --- Tab bookkeeping, with no HTTP in the way --------------------------------- +def test_a_key_with_a_colon_in_the_path_survives(): + """`split`, not `str.split`: a key that lost half its path would silently + open a different file.""" + assert canvas_service.split("agent:/srv/a:b.py") == ("agent", "/srv/a:b.py") + + +def test_one_file_has_one_key(): + """A tab a model opened and a tab a person opened must be one tab, or the + panel shows the same file twice and only one is the one being saved.""" + assert ( + canvas_service.path_key("/srv/app", "./main.py") + == canvas_service.path_key("/srv/app", "main.py") + == canvas_service.path_key("/srv/app", "/srv/app/main.py") + ) + + +def test_opening_the_same_key_twice_is_one_tab(): + state: dict = {} + canvas_service.open_tab(state, {"key": "note:1", "title": "A"}) + canvas_service.open_tab(state, {"key": "note:1", "title": "A"}) + assert len(state["tabs"]) == 1 + assert state["active"] == "note:1" + + +def test_a_model_opening_a_tab_does_not_take_the_screen(): + """An agent reads forty files in a long reply. If each one took the panel, + somebody reading the third would be dragged through the other + thirty-seven -- and anybody halfway through an edit would lose it.""" + state: dict = {} + canvas_service.open_tab(state, {"key": "note:1", "title": "Mine"}) + canvas_service.open_tab(state, {"key": "agent:/a.py", "title": "Theirs"}, activate=False) + + assert state["active"] == "note:1" + assert [t["key"] for t in state["tabs"]] == ["note:1", "agent:/a.py"] + + +def test_the_first_tab_is_activated_even_by_a_model(): + """Otherwise a panel full of tabs would have nothing in front, which reads + as a panel that failed to load.""" + state: dict = {} + canvas_service.open_tab(state, {"key": "agent:/a.py"}, activate=False) + assert state["active"] == "agent:/a.py" + + +def test_eviction_never_closes_the_tab_in_front(): + state: dict = {} + canvas_service.open_tab(state, {"key": "note:keep"}) + for index in range(canvas_service.MAX_TABS + 4): + canvas_service.open_tab(state, {"key": f"agent:/f{index}.py"}, activate=False) + + keys = [t["key"] for t in state["tabs"]] + assert len(keys) == canvas_service.MAX_TABS + assert "note:keep" in keys + assert state["active"] == "note:keep" + + +def test_closing_the_active_tab_moves_to_another(): + state: dict = {} + canvas_service.open_tab(state, {"key": "note:1"}) + canvas_service.open_tab(state, {"key": "note:2"}) + canvas_service.close_tab(state, "note:2") + assert state["active"] == "note:1" + + +def test_closing_the_last_tab_leaves_nothing_active(): + state: dict = {} + canvas_service.open_tab(state, {"key": "note:1"}) + canvas_service.close_tab(state, "note:1") + assert state["active"] == "" + assert state["tabs"] == [] + + +def test_merge_keeps_a_tab_opened_during_the_reply(): + """`_persist` is the single writer and its snapshot was seeded when the + reply began, so overwriting would drop what somebody opened since.""" + stored = {"tabs": [{"key": "note:mine", "title": "Mine"}], "active": "note:mine"} + live = {"tabs": [{"key": "agent:/a.py", "title": "Theirs"}], "active": "agent:/a.py"} + + merged = canvas_service.merge(stored, live) + assert {t["key"] for t in merged["tabs"]} == {"note:mine", "agent:/a.py"} + # And a reply finishing ten minutes later must not move what is in front. + assert merged["active"] == "note:mine" + + +# --- Through the routes -------------------------------------------------------- +def test_the_panel_opens_empty(client: TestClient, db, registered, make_chat): + _add_connection(db) + chat_id = make_chat() + response = client.get(f"/api/chats/{chat_id}/canvas") + assert response.status_code == 200 + assert "Nothing open" in response.text + + +def test_someone_elses_chat_is_a_404(client: TestClient, db, registered, make_chat): + _add_connection(db) + other = User(name="Sam", email="sam@shire.test", password_hash="x") + db.add(other) + db.commit() + chat = Chat(user_id=other.id, model_id="test-model") + db.add(chat) + db.commit() + + assert client.get(f"/api/chats/{chat.id}/canvas").status_code == 404 + assert ( + client.post(f"/api/chats/{chat.id}/canvas/tabs", data={"key": "note:1"}).status_code + == 404 + ) + + +def test_a_get_never_opens_a_tab(client: TestClient, db, registered, make_chat): + """There is no CSRF token here and the cookie is SameSite Lax, so a + state-changing GET is a link somebody can be made to follow.""" + _add_connection(db) + chat_id = make_chat() + client.get(f"/api/chats/{chat_id}/canvas?key=scratch:{chat_id}") + assert not (db.get(Chat, chat_id).canvas_json or {}).get("tabs") + + +def test_opening_and_closing_the_scratch_document(client: TestClient, db, registered, make_chat): + _add_connection(db) + chat_id = make_chat() + + opened = client.post( + f"/api/chats/{chat_id}/canvas/tabs", data={"key": f"scratch:{chat_id}"} + ) + assert opened.status_code == 200 + db.expire_all() + assert (db.get(Chat, chat_id).canvas_json or {})["active"] == f"scratch:{chat_id}" + + client.post(f"/api/chats/{chat_id}/canvas/tabs/close", data={"key": f"scratch:{chat_id}"}) + db.expire_all() + assert (db.get(Chat, chat_id).canvas_json or {})["tabs"] == [] + + +def test_a_scratch_key_naming_another_chat_is_refused( + client: TestClient, db, registered, make_chat +): + """A forged key must not reach another conversation's pad.""" + _add_connection(db) + mine = make_chat() + theirs = make_chat() + + response = client.post(f"/api/chats/{mine}/canvas/tabs", data={"key": f"scratch:{theirs}"}) + assert "another chat" in response.text + db.expire_all() + assert not (db.get(Chat, mine).canvas_json or {}).get("tabs") + + +def test_an_unknown_source_says_so_rather_than_500ing( + client: TestClient, db, registered, make_chat +): + """An exception page swapped into a side panel is a blank side panel.""" + _add_connection(db) + chat_id = make_chat() + response = client.post(f"/api/chats/{chat_id}/canvas/tabs", data={"key": "wizard:1"}) + assert response.status_code == 200 + assert "nothing to open" in response.text.lower() + + +def test_a_tab_whose_row_was_deleted_renders_an_error( + client: TestClient, db, registered, make_chat +): + _add_connection(db) + chat_id = make_chat() + user = db.get(User, db.get(Chat, chat_id).user_id) + note = notes_service.create(db, owner=user, title="Gone", body="soon") + client.post(f"/api/chats/{chat_id}/canvas/tabs", data={"key": f"note:{note.id}"}) + notes_service.delete(db, note) + + response = client.get(f"/api/chats/{chat_id}/canvas") + assert response.status_code == 200 + assert "not there any more" in response.text + + +# --- Saving -------------------------------------------------------------------- +def test_a_note_is_saved_through_the_canvas(client: TestClient, db, registered, make_chat): + _add_connection(db) + chat_id = make_chat() + user = db.get(User, db.get(Chat, chat_id).user_id) + note = notes_service.create(db, owner=user, title="Errands", body="alpha") + + doc = canvas_service._stamp(note, note.body) + response = client.post( + f"/api/chats/{chat_id}/canvas/save", + data={"key": f"note:{note.id}", "text": "beta", "revision": doc}, + ) + assert response.status_code == 200 + db.expire_all() + assert db.get(type(note), note.id).body == "beta" + + +def test_a_stale_revision_writes_nothing(client: TestClient, db, registered, make_chat): + """Never save silently over somebody else's change, and never discard what + was typed here either -- the card carries both.""" + _add_connection(db) + chat_id = make_chat() + user = db.get(User, db.get(Chat, chat_id).user_id) + note = notes_service.create(db, owner=user, title="Errands", body="alpha") + + response = client.post( + f"/api/chats/{chat_id}/canvas/save", + data={"key": f"note:{note.id}", "text": "beta", "revision": "0:999"}, + ) + assert response.status_code == 200 + assert "changed after you opened it" in response.text + # What was typed comes back in the box, so Overwrite is one click. + assert "beta" in response.text + db.expire_all() + assert db.get(type(note), note.id).body == "alpha" + + +def test_an_empty_revision_overwrites(client: TestClient, db, registered, make_chat): + """Which is exactly what Overwrite on the conflict card sends: somebody has + been shown both versions and chosen.""" + _add_connection(db) + chat_id = make_chat() + user = db.get(User, db.get(Chat, chat_id).user_id) + note = notes_service.create(db, owner=user, title="Errands", body="alpha") + + client.post( + f"/api/chats/{chat_id}/canvas/save", + data={"key": f"note:{note.id}", "text": "beta", "revision": ""}, + ) + db.expire_all() + assert db.get(type(note), note.id).body == "beta" + + +def test_someone_elses_note_cannot_be_saved(client: TestClient, db, registered, make_chat): + """Sharing grants reading only.""" + _add_connection(db) + chat_id = make_chat() + other = User(name="Sam", email="sam@shire.test", password_hash="x") + db.add(other) + db.commit() + note = notes_service.create(db, owner=other, title="Theirs", body="alpha") + + response = client.post( + f"/api/chats/{chat_id}/canvas/save", + data={"key": f"note:{note.id}", "text": "beta", "revision": ""}, + ) + db.expire_all() + assert db.get(type(note), note.id).body == "alpha" + assert "not there any more" in response.text or "not yours" in response.text + + +def test_an_attachment_has_no_save_path(client: TestClient, db, registered, make_chat): + """`DELETE /api/files/{id}` already refuses once an attachment has been sent + because it would rewrite a message somebody read. Editing is the same act + with a quieter failure.""" + _add_connection(db) + chat_id = make_chat() + attachment = Attachment( + user_id=db.get(Chat, chat_id).user_id, + chat_id=chat_id, + filename="notes.txt", + stored_name="x.txt", + media_type="text/plain", + kind="text", + extracted_text="alpha", + ) + db.add(attachment) + db.commit() + + response = client.post( + f"/api/chats/{chat_id}/canvas/save", + data={"key": f"file:{attachment.id}", "text": "beta", "revision": ""}, + ) + assert "only be read" in response.text + db.expire_all() + assert db.get(Attachment, attachment.id).extracted_text == "alpha" + + +def test_an_attachment_from_another_chat_is_refused( + client: TestClient, db, registered, make_chat +): + """A canvas must not browse another conversation's files by id.""" + _add_connection(db) + mine = make_chat() + theirs = make_chat() + attachment = Attachment( + user_id=db.get(Chat, theirs).user_id, + chat_id=theirs, + filename="notes.txt", + stored_name="x.txt", + media_type="text/plain", + kind="text", + extracted_text="alpha", + ) + db.add(attachment) + db.commit() + + response = client.post( + f"/api/chats/{mine}/canvas/tabs", data={"key": f"file:{attachment.id}"} + ) + assert "another chat" in response.text + + +# --- The agent source, without a machine ------------------------------------------ +def test_an_ordinary_chat_cannot_open_a_project_file( + client: TestClient, db, registered, make_chat +): + _add_connection(db) + chat_id = make_chat() + response = client.post( + f"/api/chats/{chat_id}/canvas/tabs", data={"key": "agent:/etc/passwd"} + ) + assert "no connection" in response.text + db.expire_all() + assert not (db.get(Chat, chat_id).canvas_json or {}).get("tabs") + + +def test_an_agent_chat_without_the_permission_cannot_either( + client: TestClient, db, registered, make_chat +): + """Re-derived server-side on every request; the template flag is decoration.""" + _add_connection(db) + chat_id = make_chat() + chat = db.get(Chat, chat_id) + chat.kind = KIND_AGENT + chat.ssh_profile_id = "nothing" + db.commit() + settings_store.update(db, {"enabled": True}, key=settings_store.AGENTS) + user = db.get(User, chat.user_id) + user.role = "user" + settings_store.update(db, {"default_permissions": {"tools.agent": False}}) + db.commit() + + assert canvas_service.agent_ready(db, user, chat) is None + + +# --- The document source ------------------------------------------------------------ +def test_a_documents_text_can_be_replaced(client: TestClient, db, registered, make_chat): + """Replacing a failed extraction by hand is the main reason to want this.""" + _add_connection(db) + chat_id = make_chat() + user = db.get(User, db.get(Chat, chat_id).user_id) + base = documents_service.create_base(db, owner=user, name="Contracts") + document = _page(db, user, base, "alpha") + document.extraction_error = "Could not read this." + db.commit() + + documents_service.set_text(db, document, "beta") + db.expire_all() + refreshed = documents_service.get(db, document.id, user) + assert refreshed.extracted_text == "beta" + # The old apology beside the new text would be the page contradicting itself. + assert refreshed.extraction_error == "" + + +def test_editing_a_document_does_not_change_a_transcript(db, registered, make_chat): + """`files.copy_document` copies the text when a document is attached, so an + edit only changes what future searches find.""" + from lembas.services import files as files_service + + _add_connection(db) + chat_id = make_chat() + user = db.get(User, db.get(Chat, chat_id).user_id) + base = documents_service.create_base(db, owner=user, name="Contracts") + document = _page(db, user, base, "alpha") + attachment = files_service.copy_document( + db, user_id=user.id, chat_id=chat_id, document=document + ) + + documents_service.set_text(db, document, "beta") + db.expire_all() + assert db.get(Attachment, attachment.id).extracted_text == "alpha" + + +# --- The scratch document ----------------------------------------------------------- +def test_the_pad_is_made_once_per_chat(db, registered, make_chat): + _add_connection(db) + chat = db.get(Chat, make_chat()) + + first = scratch_service.for_chat(db, chat) + second = scratch_service.for_chat(db, chat) + assert first.id == second.id + assert db.scalars(select(ScratchDoc)).all() == [first] + + +def test_asking_whether_there_is_one_does_not_make_one(db, registered, make_chat): + """Otherwise every chat ever opened acquires an empty row.""" + _add_connection(db) + chat = db.get(Chat, make_chat()) + assert scratch_service.get(db, chat) is None + assert db.scalars(select(ScratchDoc)).all() == [] + + +def test_appending_twice_keeps_both(db, registered, make_chat): + """A read-and-concatenate at the call site would let two calls in one round + each read the same body, and the second would drop the first.""" + _add_connection(db) + chat = db.get(Chat, make_chat()) + doc = scratch_service.for_chat(db, chat) + + scratch_service.append(db, doc, "first", author="model") + scratch_service.append(db, doc, "second", author="model") + assert "first" in doc.body + assert "second" in doc.body + + +def test_the_pad_keeps_trailing_whitespace(db, registered, make_chat): + """A save that silently trims the line you are standing on is the kind of + thing that makes an editor feel broken.""" + _add_connection(db) + chat = db.get(Chat, make_chat()) + doc = scratch_service.for_chat(db, chat) + scratch_service.update(db, doc, body="a line \n") + assert doc.body == "a line \n" + + +def test_attaching_the_pad_copies_it(client: TestClient, db, registered, make_chat): + """The pad goes on being written after the message is sent, by both sides.""" + _add_connection(db) + chat_id = make_chat() + chat = db.get(Chat, chat_id) + doc = scratch_service.for_chat(db, chat) + scratch_service.update(db, doc, body="the draft") + + response = client.post("/api/files/from-scratch", data={"chat_id": chat_id}) + assert response.status_code == 200 + + scratch_service.update(db, doc, body="changed since") + db.expire_all() + attachment = db.scalar(select(Attachment)) + assert attachment.extracted_text == "the draft" + + +def test_an_empty_pad_is_not_worth_attaching(client: TestClient, db, registered, make_chat): + _add_connection(db) + chat_id = make_chat() + response = client.post("/api/files/from-scratch", data={"chat_id": chat_id}) + assert "not available" in response.text + assert db.scalar(select(Attachment)) is None + + +def test_the_pad_of_another_chat_cannot_be_attached( + client: TestClient, db, registered, make_chat +): + _add_connection(db) + other = User(name="Sam", email="sam@shire.test", password_hash="x") + db.add(other) + db.commit() + chat = Chat(user_id=other.id, model_id="test-model") + db.add(chat) + db.commit() + scratch_service.update(db, scratch_service.for_chat(db, chat), body="theirs") + + response = client.post("/api/files/from-scratch", data={"chat_id": chat.id}) + assert "not available" in response.text + assert db.scalar(select(Attachment)) is None + + +# --- Where the panel appears --------------------------------------------------------- +def test_the_panel_is_on_a_chat_and_not_on_the_new_chat_screen( + client: TestClient, db, registered, make_chat +): + """Absent before there is a row, for the reason the scope menu is: there is + nothing to hang a tab on yet.""" + _add_connection(db) + assert 'id="canvas"' not in client.get("/chat").text + assert 'id="canvas"' in client.get(f"/chat/{make_chat()}").text + + +def test_the_panel_shares_one_slot_with_the_others( + client: TestClient, db, registered, make_chat +): + """At 1280px the sidebar plus two panels leaves about seventy pixels of + conversation, so only one of the three is ever open.""" + _add_connection(db) + page = client.get(f"/chat/{make_chat()}").text + assert 'data-toggle="#canvas" data-toggle-group="side"' in page + + +@pytest.mark.parametrize("verb", ["get"]) +def test_the_tab_routes_refuse_the_wrong_method( + client: TestClient, db, registered, make_chat, verb +): + """A control wired to a method its route does not serve fails silently.""" + _add_connection(db) + chat_id = make_chat() + assert getattr(client, verb)(f"/api/chats/{chat_id}/canvas/tabs").status_code == 405 + assert getattr(client, verb)(f"/api/chats/{chat_id}/canvas/save").status_code == 405 diff --git a/tests/test_canvas_ssh.py b/tests/test_canvas_ssh.py new file mode 100644 index 0000000..728318a --- /dev/null +++ b/tests/test_canvas_ssh.py @@ -0,0 +1,176 @@ +"""Reading and writing a file for somebody who is about to edit it. + +The model-facing `read_file`/`write_file` pair is deliberately untouched: what +they return is a contract a model has been shown, and it is the right contract +for a model. It is the wrong one here, and these are the cases that say why. +""" + +from __future__ import annotations + +import pytest + +from lembas.services.agent import ssh as ssh_service +from lembas.services.agent.base import Conflict, ExecError + +asyncssh = pytest.importorskip("asyncssh") + + +class _Server(asyncssh.SSHServer): + def begin_auth(self, username: str) -> bool: + return False + + +@pytest.fixture +async def machine(tmp_path): + project = tmp_path / "project" + project.mkdir() + server = await asyncssh.create_server( + _Server, + "127.0.0.1", + 0, + server_host_keys=[asyncssh.generate_private_key("ssh-ed25519")], + sftp_factory=True, + ) + port = next(iter(server.sockets)).getsockname()[1] + line, fingerprint = await ssh_service.capture_host_key("127.0.0.1", port) + try: + yield {"port": port, "host_key": line, "dir": str(project), "path": project} + finally: + server.close() + await server.wait_closed() + + +def _executor(machine) -> ssh_service.SshExecutor: + return ssh_service.SshExecutor( + { + "host": "127.0.0.1", + "port": machine["port"], + "username": "tester", + "auth": "password", + "credential": "", + "host_key": machine["host_key"], + }, + machine["dir"], + ) + + +# --- Fidelity ------------------------------------------------------------------ +async def test_an_escape_sequence_survives_a_round_trip(machine): + """The whole reason this is not `read_file`. That one ends in + `clean_output`, which strips ANSI escapes -- right for the output of a + command, and here it means opening a file and pressing Save rewrites it + with the escapes gone.""" + original = "red \x1b[31mtext\x1b[0m here\n" + (machine["path"] / "colours.txt").write_text(original) + executor = _executor(machine) + + opened = await executor.read_text("colours.txt") + assert opened.text == original + + await executor.write_text("colours.txt", opened.text, if_unchanged=opened.revision) + assert (machine["path"] / "colours.txt").read_text() == original + + +async def test_the_model_facing_read_still_strips_them(machine): + """Pinned as a pair: the contract a model was shown has not moved.""" + (machine["path"] / "colours.txt").write_text("red \x1b[31mtext\x1b[0m here\n") + text = await _executor(machine).read_file("colours.txt") + assert "\x1b[31m" not in text + + +async def test_undecodable_bytes_are_reported_rather_than_replaced(machine): + """errors="replace" would hand back U+FFFD for every one of them, and + saving that back is how a file is quietly destroyed.""" + (machine["path"] / "blob.bin").write_bytes(b"\xff\xfe\x00\x01binary") + opened = await _executor(machine).read_text("blob.bin") + assert opened.binary is True + assert opened.text == "" + + +async def test_a_nul_byte_early_on_reads_as_binary(machine): + (machine["path"] / "blob.bin").write_bytes(b"text\x00more text") + assert (await _executor(machine).read_text("blob.bin")).binary is True + + +async def test_utf8_beyond_ascii_is_not_binary(machine): + (machine["path"] / "note.txt").write_text("a mallorn tree — Lothlórien\n") + opened = await _executor(machine).read_text("note.txt") + assert opened.binary is False + assert "Lothlórien" in opened.text + + +# --- Size ------------------------------------------------------------------------ +async def test_a_large_file_opens_truncated(machine): + (machine["path"] / "big.log").write_text("x" * (ssh_service.MAX_READ_BYTES + 500)) + opened = await _executor(machine).read_text("big.log") + assert opened.truncated is True + assert len(opened.text) == ssh_service.MAX_READ_BYTES + + +async def test_an_oversize_write_is_refused_not_truncated(machine): + """`write_file` truncates because a model is told how many bytes it wrote. + Somebody pressing Save would lose the tail with nothing said.""" + executor = _executor(machine) + (machine["path"] / "big.txt").write_text("small") + + with pytest.raises(ExecError, match="Nothing was written"): + await executor.write_text("big.txt", "y" * (ssh_service.MAX_WRITE_BYTES + 1)) + + assert (machine["path"] / "big.txt").read_text() == "small" + + +# --- Conflict --------------------------------------------------------------------- +async def test_a_file_that_moved_underneath_refuses_the_save(machine): + import os + + target = machine["path"] / "note.txt" + target.write_text("alpha\n") + executor = _executor(machine) + opened = await executor.read_text("note.txt") + + # Somebody else's editor, a build, a checkout. The size differs, so this + # does not depend on the filesystem's mtime resolution. + target.write_text("something else entirely\n") + os.utime(target, (0, 0)) + + with pytest.raises(Conflict): + await executor.write_text("note.txt", "beta\n", if_unchanged=opened.revision) + + assert target.read_text() == "something else entirely\n" + + +async def test_a_save_with_no_token_overwrites(machine): + """Which is what Overwrite on the conflict card does.""" + target = machine["path"] / "note.txt" + target.write_text("alpha\n") + await _executor(machine).write_text("note.txt", "beta\n") + assert target.read_text() == "beta\n" + + +async def test_a_new_file_can_be_created(machine): + """Open a path that is not there, type, Save. The stat finds nothing and + there is nothing for the token to disagree with.""" + executor = _executor(machine) + await executor.write_text("fresh.txt", "hello\n", if_unchanged="0:0") + assert (machine["path"] / "fresh.txt").read_text() == "hello\n" + + +async def test_the_revision_moves_after_a_write(machine): + """Or the second save from the same tab would always conflict.""" + target = machine["path"] / "note.txt" + target.write_text("alpha\n") + executor = _executor(machine) + + opened = await executor.read_text("note.txt") + written = await executor.write_text( + "note.txt", "much longer contents\n", if_unchanged=opened.revision + ) + assert written.revision != opened.revision + + await executor.write_text("note.txt", "again\n", if_unchanged=written.revision) + assert target.read_text() == "again\n" + + +async def test_reading_something_that_is_not_there_says_so(machine): + with pytest.raises(ExecError, match="no file"): + await _executor(machine).read_text("nowhere.txt") diff --git a/tests/test_canvas_stream.py b/tests/test_canvas_stream.py new file mode 100644 index 0000000..0905989 --- /dev/null +++ b/tests/test_canvas_stream.py @@ -0,0 +1,218 @@ +"""A file the model opened, reaching the panel. + +The rule this pins is the one that would fail silently: the `canvas` frame is +guarded on truthiness, so it can never blank itself. An empty one would close +every tab somebody had open -- the "approval card you could press twice" failure +with the sign reversed. +""" + +from __future__ import annotations + +import json + +from sqlalchemy import select + +from lembas.db.models import ROLE_ASSISTANT, Chat, Connection, Message, Model +from lembas.services import canvas as canvas_service +from lembas.services import generation as generation_service + + +def _chat_with_a_reply(db, user_id): + connection = Connection(name="c", base_url="http://127.0.0.1:1", api_key_encrypted="") + db.add(connection) + db.commit() + db.add(Model(connection_id=connection.id, model_id="m")) + db.commit() + chat = Chat(user_id=user_id, model_id="m", connection_id=connection.id) + db.add(chat) + db.commit() + db.add(Message(chat_id=chat.id, role="user", content="Have a look", complete=True)) + db.commit() + assistant = Message(chat_id=chat.id, role=ROLE_ASSISTANT, content="", complete=False) + db.add(assistant) + db.commit() + return chat.id, assistant.id + + +def _stub_stream(text: str): + async def stream_chat(_endpoint, _payload): + yield {"choices": [{"delta": {"content": text}}]} + + return stream_chat + + +# --- The frame ------------------------------------------------------------------ +def test_the_frame_is_absent_when_nothing_was_opened(db, user_id): + """Asserted directly, because this is the whole safety property. `reasoning`, + `tools` and `render` are guarded the same way; `metrics`, `status` and `ask` + are not, because each of *those* has to be able to clear.""" + generation = generation_service.Generation(chat_id="x", message_id="y") + assert not generation.canvas.get("tabs") + + +def test_the_frame_carries_the_whole_strip(db, user_id): + """Not a delta. A follower attaching mid-reply has no earlier fragments to + append to, so it gets every tab the reply has touched.""" + from lembas.api.chats import _canvas_tabs + + state: dict = {} + canvas_service.open_tab(state, {"key": "agent:/a.py", "title": "a.py"}, activate=False) + canvas_service.open_tab(state, {"key": "agent:/b.py", "title": "b.py"}, activate=False) + + html = _canvas_tabs("chat-1", state) + assert "a.py" in html + assert "b.py" in html + # Out of band, because it belongs to a panel and not to the bubble the + # stream is writing into. + assert 'hx-swap-oob="true"' in html + assert 'id="canvas-tabs"' in html + + +def test_a_path_with_a_quote_does_not_break_the_strip(): + """The key goes into an hx-vals attribute. `| tojson` rather than quoting by + hand, or a file called `"` produces vals that do not parse and the tab + silently stops working.""" + from lembas.api.chats import _canvas_tabs + + state: dict = {} + canvas_service.open_tab(state, {"key": 'agent:/srv/a"b.py', "title": 'a"b.py'}) + html = _canvas_tabs("chat-1", state) + assert 'a\\"b.py' in html or "a"b.py" in html + + +# --- Through the loop ------------------------------------------------------------- +async def test_two_reads_in_one_round_both_land(db, user_id, monkeypatch): + """Seeded once and mutated, not re-read per call: two `file_read`s that each + read the row would leave only the second.""" + generation = generation_service.Generation(chat_id="x", message_id="y") + generation.canvas = {"tabs": [], "active": ""} + + for path in ("/srv/a.py", "/srv/b.py"): + canvas_service.open_tab( + generation.canvas, {"key": f"agent:{path}", "title": path}, activate=False + ) + + assert [t["key"] for t in generation.canvas["tabs"]] == [ + "agent:/srv/a.py", + "agent:/srv/b.py", + ] + + +async def test_a_reply_writes_its_tabs_onto_the_chat(db, user_id, monkeypatch): + chat_id, message_id = _chat_with_a_reply(db, user_id) + monkeypatch.setattr(generation_service, "stream_chat", _stub_stream("Looked.")) + + generation = generation_service.Generation(chat_id=chat_id, message_id=message_id) + await generation_service._run(generation) + # Nothing was opened, so nothing is written -- and in particular the column + # is not blanked. + db.expire_all() + assert not (db.get(Chat, chat_id).canvas_json or {}).get("tabs") + + +async def test_a_reply_never_wipes_the_tabs_that_were_already_open( + db, user_id, monkeypatch +): + """The snapshot is seeded from the row when the reply begins, so a reply + that opens nothing writes nothing -- and one that opens something adds to + what was there rather than replacing it.""" + chat_id, message_id = _chat_with_a_reply(db, user_id) + monkeypatch.setattr(generation_service, "stream_chat", _stub_stream("Looked.")) + + chat = db.get(Chat, chat_id) + chat.canvas_json = canvas_service.open_tab({}, {"key": f"scratch:{chat_id}"}) + db.commit() + + generation = generation_service.Generation(chat_id=chat_id, message_id=message_id) + await generation_service._run(generation) + + db.expire_all() + stored = db.get(Chat, chat_id).canvas_json or {} + assert [t["key"] for t in stored["tabs"]] == [f"scratch:{chat_id}"] + assert stored["active"] == f"scratch:{chat_id}" + + +async def test_the_snapshot_is_seeded_from_the_row(db, user_id, monkeypatch): + """Seeded once where the chat is already loaded, rather than re-read per + call -- which is what lets two file reads in one round both land.""" + chat_id, message_id = _chat_with_a_reply(db, user_id) + monkeypatch.setattr(generation_service, "stream_chat", _stub_stream("Looked.")) + + chat = db.get(Chat, chat_id) + chat.canvas_json = canvas_service.open_tab({}, {"key": f"scratch:{chat_id}"}) + db.commit() + + generation = generation_service.Generation(chat_id=chat_id, message_id=message_id) + await generation_service._run(generation) + + assert [t["key"] for t in generation.canvas["tabs"]] == [f"scratch:{chat_id}"] + + +# --- What the runners write --------------------------------------------------------- +def test_the_file_tools_name_the_key_the_same_way_a_person_would(): + """A tab a model opened and one a person opened have to be one tab.""" + from lembas.services.agent import tools as agent_tools + from lembas.services.agent.session import AgentContext + + agent = AgentContext(chat_id="c", label="Box", project_dir="/srv/app") + for spelling in ("./main.py", "main.py", "/srv/app/main.py"): + assert agent_tools._canvas(agent, spelling)["key"] == "agent:/srv/app/main.py" + + +def test_the_key_matches_the_read_path_set(): + """Both come from `_path_key`. If they could drift, `file_edit`'s "read it + first" and the canvas would disagree about which file was read.""" + from lembas.services.agent import tools as agent_tools + from lembas.services.agent.session import AgentContext + + agent = AgentContext(chat_id="c", label="Box", project_dir="/srv/app") + assert agent_tools._canvas(agent, "./main.py")["key"] == ( + f"agent:{agent_tools._path_key(agent, './main.py')}" + ) + + +def test_the_swap_target_exists_on_a_streaming_bubble(): + """A frame with nowhere to land is a frame that silently does nothing.""" + from lembas.web.templating import templates + + html = templates.get_template("chat/_message.html").render( + { + "message": Message(id="m1", chat_id="c1", role=ROLE_ASSISTANT, content=""), + "streaming": True, + "chat": None, + "user": None, + "models_by_id": {}, + "bodies": {}, + } + ) + assert 'sse-swap="canvas"' in html + + +def test_scratch_write_opens_its_tab(db, user_id): + """It rides on the same mechanism as the file tools, and for the same + reason: no new schema and no tokens.""" + from lembas.services.tools import REGISTRY + + tool = REGISTRY["scratch_write"] + assert tool.family == "scratch" + # RISK_READ, on plan_update's argument: risk is what a tool does to the + # world the four modes govern, which is the machine. + assert tool.risk == "read" + + +def test_the_event_survives_into_the_stored_transcript(): + """Harmless and mildly useful: `_tool_activity.html` reads named keys.""" + event = {"name": "file_read", "canvas": {"key": "agent:/a.py"}} + assert json.loads(json.dumps(event))["canvas"]["key"] == "agent:/a.py" + + +def test_nothing_but_the_chat_row_holds_the_tabs(db, user_id): + """No table, no cleanup path: the tabs go when the chat does.""" + chat_id, _ = _chat_with_a_reply(db, user_id) + chat = db.get(Chat, chat_id) + chat.canvas_json = canvas_service.open_tab({}, {"key": "note:1"}) + db.commit() + + db.delete(chat) + db.commit() + assert db.scalar(select(Chat)) is None diff --git a/tests/test_layout_bounds.py b/tests/test_layout_bounds.py new file mode 100644 index 0000000..8c8d389 --- /dev/null +++ b/tests/test_layout_bounds.py @@ -0,0 +1,74 @@ +"""Panel widths: three numbers per panel, in three files, that must agree. + +`set_layout` drops a CSS variable it does not recognise, and it drops it +silently -- an older browser sending a key a newer release removed must not fail +the whole request. The cost of that kindness is that a panel whose width is +missing from `LAYOUT_BOUNDS` is one whose drag handle appears to work, moves the +edge, and forgets by the next page load. Nothing anywhere says so. + +So the three are pinned here: the allowlist entry, the `data-resize-min` on the +handle, and the `--*-width-min` token the CSS clamps with. +""" + +from __future__ import annotations + +import re +from pathlib import Path + +import lembas +from lembas.api.preferences import LAYOUT_BOUNDS + +ROOT = Path(lembas.__file__).parent +TOKENS = (ROOT / "web/static/css/tokens.css").read_text(encoding="utf-8") +TEMPLATES = ROOT / "web/templates" + +# The panels with a drag handle, and the template each handle lives in. +PANELS = { + "--terminal-width": "chat/_terminal.html", + "--canvas-width": "chat/_canvas.html", +} + +# 1rem, everywhere in this application. +REM = 16 + + +def _resize_min(template: str) -> int: + text = (TEMPLATES / template).read_text(encoding="utf-8") + found = re.search(r'data-resize-min="(\d+)"', text) + assert found, f"{template} has a resize handle with no minimum" + return int(found.group(1)) + + +def _token_min(name: str) -> int: + found = re.search(rf"{re.escape(name)}-min:\s*([\d.]+)rem", TOKENS) + assert found, f"{name}-min is not declared in tokens.css" + return int(float(found.group(1)) * REM) + + +def test_every_dragged_panel_is_in_the_allowlist(): + """Without the entry the drag is silently discarded on the way to the + account, so the width survives in one browser and vanishes in the next.""" + missing = [name for name in PANELS if name not in LAYOUT_BOUNDS] + assert not missing, f"not in LAYOUT_BOUNDS: {missing}" + + +def test_the_three_minimums_agree(): + for name, template in PANELS.items(): + assert LAYOUT_BOUNDS[name][0] == _resize_min(template) == _token_min(name), name + + +def test_no_bound_lets_a_panel_become_unreachable(): + """A width outside these is a panel somebody cannot see well enough to drag + back, which is the other half of what the allowlist is for.""" + for name, (low, high) in LAYOUT_BOUNDS.items(): + assert 0 < low < high, name + + +def test_the_canvas_starts_wider_than_the_terminal(): + """A source line is longer than eighty columns once nothing is re-wrapping + it, and this one holds prose as well.""" + widths = { + name: float(re.search(rf"{re.escape(name)}:\s*([\d.]+)rem", TOKENS).group(1)) + for name in PANELS + } + assert widths["--canvas-width"] > widths["--terminal-width"]