diff --git a/src/lembas/api/admin_agents.py b/src/lembas/api/admin_agents.py index 20806bf..108e26b 100644 --- a/src/lembas/api/admin_agents.py +++ b/src/lembas/api/admin_agents.py @@ -72,6 +72,8 @@ async def save_agents( terminal_idle_timeout: int = Form(1800), terminal_max_sessions: int = Form(20), terminal_max_per_user: int = Form(3), + index_enabled: bool = Form(False), + index_chars: int = Form(2000), ) -> Response: settings_store.update( db, @@ -94,6 +96,11 @@ async def save_agents( "terminal_idle_timeout": min(max(terminal_idle_timeout, 60), 86400), "terminal_max_sessions": min(max(terminal_max_sessions, 1), 500), "terminal_max_per_user": min(max(terminal_max_per_user, 1), 50), + "index_enabled": index_enabled, + # Zero is kept rather than clamped up: it means "list the + # directory for the file picker but put none of it in the + # prompt", which nothing else can say. + "index_chars": min(max(index_chars, 0), 20_000), }, key=settings_store.AGENTS, ) diff --git a/src/lembas/api/agents.py b/src/lembas/api/agents.py index 9c1dbe5..d180c46 100644 --- a/src/lembas/api/agents.py +++ b/src/lembas/api/agents.py @@ -24,6 +24,7 @@ from lembas.api.deps import Db, RequiredUser, require_permission from lembas.api.pages import sidebar_context from lembas.db.models import AUTH_METHODS, AUTH_PASSWORD, SshProfile from lembas.services import settings_store +from lembas.services.agent import index as index_service from lembas.services.agent import ssh as ssh_service from lembas.services.agent import terminal as terminal_service from lembas.services.agent.base import ExecError @@ -360,6 +361,7 @@ async def forget_host_key(request: Request, db: Db, user: RequiredUser, profile_ # Un-trusting a host has to reach the shell already open on it, or the one # connection that matters is the one this does not touch. await terminal_service.close_for_profile(profile.id) + index_service.forget(profile.id) db.commit() return render(request, "agents/_check.html", {"profile": profile, "forgotten": True}) @@ -369,6 +371,7 @@ async def delete_profile(db: Db, user: RequiredUser, profile_id: str) -> Respons profile = _profile(db, user, profile_id) name = profile.name await terminal_service.close_for_profile(profile.id) + index_service.forget(profile.id) db.delete(profile) db.commit() log.info("%s deleted ssh profile %s", user.email, name) @@ -416,6 +419,7 @@ async def update_profile(request: Request, db: Db, user: RequiredUser, profile_i # would simply not be true of the terminal on screen. if not profile.enabled or not profile.host_key or (profile.host, profile.port) != before: await terminal_service.close_for_profile(profile.id) + index_service.forget(profile.id) db.commit() return RedirectResponse( diff --git a/src/lembas/api/chats.py b/src/lembas/api/chats.py index 10f3ec3..000dacf 100644 --- a/src/lembas/api/chats.py +++ b/src/lembas/api/chats.py @@ -244,6 +244,53 @@ async def inspect_chat(request: Request, db: Db, user: RequiredUser, chat_id: st ) +@router.get("/{chat_id}/usage") +async def chat_usage(request: Request, db: Db, user: RequiredUser, chat_id: str) -> Response: + """What this conversation has cost, and how full the window is. + + Owner-checked and nothing else: it is your own chat's totals. Unlike the + inspector next door there is no admin branch, because there is no reason + for one -- the numbers describe a conversation, and reading somebody's + conversation is exactly what `sharing` has no admin branch for either. + + Summed from what each reply recorded rather than recomputed: an endpoint + that reported no usage contributed an estimate at the time, and re-deriving + it now with a different estimator would make the totals move under a chat + that had not changed. + """ + chat = _owned_chat(db, chat_id, user.id) + replies = list( + db.scalars( + select(Message) + .where(Message.chat_id == chat.id, Message.role == ROLE_ASSISTANT) + .order_by(Message.created_at) + ) + ) + + totals = {"prompt": 0, "completion": 0, "total": 0} + estimated = False + for reply in replies: + usage = metrics_service.from_message(reply.usage_json) + totals["prompt"] += usage.prompt_tokens + totals["completion"] += usage.completion_tokens + totals["total"] += usage.total_tokens + estimated = estimated or usage.estimated + + last = replies[-1] if replies else None + return render( + request, + "chat/_usage.html", + { + "chat": chat, + "totals": totals, + "estimated": estimated, + "replies": len(replies), + "metrics": metrics_service.from_message(last.usage_json if last else None), + "model": chat_service.model_for(db, chat), + }, + ) + + # Roughly what a downscaled phone photo comes to as base64. The exact figure # does not matter; putting megabytes of it into the DOM does. _REDACTED_URI = "data:…base64 image omitted…" diff --git a/src/lembas/api/files.py b/src/lembas/api/files.py index 0303642..e06bc19 100644 --- a/src/lembas/api/files.py +++ b/src/lembas/api/files.py @@ -19,6 +19,7 @@ from fastapi.responses import FileResponse from lembas.api.deps import Db, RequiredUser, require_permission from lembas.db.models import Attachment, Document +from lembas.security import permissions from lembas.services import files as files_service from lembas.services import settings_store from lembas.services.fetch import FetchError, fetch @@ -167,6 +168,146 @@ async def knowledge_picker( ) +@router.get("/mention-picker", dependencies=[Depends(require_permission("files.upload"))]) +async def mention_picker( + request: Request, + db: Db, + user: RequiredUser, + q: str = "", + chat_id: str = "", + profile_id: str = "", + project_dir: str = "", +) -> Response: + """What `@` offers: files under the project directory, and the library. + + One menu from two sources, because a person typing `@readme` is not + thinking about which store the answer lives in. The project half is only + there for an agent chat and only when a listing has already been built -- + this is a keystroke-latency path and it must never wait on a machine. + + Filtered server-side, like the knowledge picker beside it and for the same + reason: the library is searched with FTS rather than filtered in the + browser, which is what makes it work at five hundred documents. The project + half is filtered here too, so the client stays one `fetch` and a list. + """ + needle = q.strip().lower() + + files: list[dict] = [] + if profile_id and permissions.has(db, user, "tools.agent"): + from lembas.db.models import SshProfile + from lembas.services.agent import index as index_service + + profile = db.get(SshProfile, profile_id) + # Re-checked rather than trusted from the query string: an id in a URL + # is not an authorisation, and this lists somebody's machine. + if profile is not None and profile.owner_id == user.id: + found = index_service.cached(profile_id, project_dir or profile.default_dir) + if found is not None: + files = [ + {"path": path, "name": path.rstrip("/").rsplit("/", 1)[-1]} + for path in found.paths + if not needle or needle in path.lower() + ][:20] + + documents: list = [] + if permissions.has(db, user, "library.use"): + if needle: + documents = documents_service.search(db, user, q, limit=10) + else: + documents = list( + db.scalars( + documents_service.visible(db, user) + .order_by(Document.created_at.desc()) + .limit(10) + ) + ) + + return templates.TemplateResponse( + request, + "chat/_mention_picker.html", + { + "request": request, + "user": user, + "files": files, + "documents": documents, + "q": q, + "chat_id": chat_id, + "profile_id": profile_id, + }, + ) + + +@router.post("/from-project", dependencies=[Depends(require_permission("files.upload"))]) +async def attach_from_project( + request: Request, + db: Db, + user: RequiredUser, + profile_id: str = Form(""), + path: str = Form(""), + chat_id: str = Form(""), +) -> Response: + """Pull one file off the far machine and attach it to this message. + + Its contents, not a reference: a model that has to spend a round calling + `file_read` often does not bother, and on a plain chat there is no + `file_read` to call. The path and the machine travel with it, so the model + is told exactly which file it is looking at rather than a bare basename it + cannot act on. + + A directory attaches its listing instead of refusing -- "@ that folder" is + a reasonable thing to mean, and the listing is what it means. + """ + from lembas.db.models import SshProfile + from lembas.services.agent import ssh as ssh_service + from lembas.services.agent.base import ExecError + + def _failed(message: str) -> Response: + return templates.TemplateResponse( + request, + "chat/_attachment_error.html", + {"request": request, "filename": path or "file", "error": message}, + ) + + if not permissions.has(db, user, "tools.agent"): + return _failed("You do not have access to connections.") + + profile = db.get(SshProfile, profile_id) + if profile is None or profile.owner_id != user.id or not profile.enabled: + return _failed("That connection is not available.") + if hint := ssh_service.available(): + return _failed(hint) + + wanted = path.strip() + if not wanted: + return _failed("No file was named.") + + executor = ssh_service.SshExecutor(ssh_service.spec_from(profile), profile.default_dir) + try: + if wanted.endswith("/"): + names = await executor.list_dir(wanted.rstrip("/")) + body = "\n".join(names) + truncated = len(names) >= ssh_service.MAX_ENTRIES + else: + body = await executor.read_file(wanted, max_bytes=ssh_service.MAX_READ_BYTES) + truncated = len(body.encode("utf-8", "ignore")) >= ssh_service.MAX_READ_BYTES + except ExecError as exc: + return _failed(exc.message) + + attachment = files_service.store_text( + db, + user_id=user.id, + chat_id=chat_id or None, + filename=wanted.rstrip("/").rsplit("/", 1)[-1] or wanted, + text=body, + truncated=truncated, + source_path=wanted, + source_label=profile.name, + ) + return templates.TemplateResponse( + request, "chat/_attachment_chip.html", {"request": request, "attachment": attachment} + ) + + @router.delete("/{attachment_id}") async def remove(db: Db, user: RequiredUser, attachment_id: str) -> Response: """Detach a file before it has been sent.""" diff --git a/src/lembas/db/models/attachment.py b/src/lembas/db/models/attachment.py index af7bc99..1ca3d77 100644 --- a/src/lembas/db/models/attachment.py +++ b/src/lembas/db/models/attachment.py @@ -55,6 +55,19 @@ class Attachment(UUIDPrimaryKey, Timestamps, Base): # is not left wondering why the model ignored it. extraction_error: Mapped[str] = mapped_column(Text, default="") + # Where this came from, when it came from somewhere with an address. + # + # `filename` is a display name and is frequently just the basename, which + # is not enough: a model told it has been given `main.py` cannot tell which + # of four it is looking at, and cannot name the file back to you if you ask + # it to change something. So a project file carries its absolute path and + # the machine it was read from, and both go into the tag the model sees. + # + # Nullable, and empty for an ordinary upload -- a file dragged in from a + # laptop has no address this instance could meaningfully report. + source_path: Mapped[str] = mapped_column(String(1000), default="") + source_label: Mapped[str] = mapped_column(String(200), default="") + message: Mapped[Message] = relationship(back_populates="attachments") # noqa: F821 @property diff --git a/src/lembas/services/agent/index.py b/src/lembas/services/agent/index.py new file mode 100644 index 0000000..8287322 --- /dev/null +++ b/src/lembas/services/agent/index.py @@ -0,0 +1,507 @@ +"""What is in a project directory, for the picker and for the model. + +Two things want this list. The `@` picker needs something to filter, and a +model working in a directory should know roughly what is in it rather than +spending its first two rounds finding out. Both want the same walk, so it +happens once and is cached. + +**Three ways of getting it, in order.** `git ls-files` first, because most +project directories are repositories and it applies `.gitignore` for free -- +without which the answer for a Node project is forty thousand paths under +`node_modules`. Then `find`, with the usual noise pruned by hand. Then a +recursive SFTP walk, which always works and costs a round trip per directory. + +**Two commands run here, and neither goes through `agent/policy.py`.** That is +deliberate and it is the same argument the terminal panel and the directory +browser rest on: this is LLeMbas listing a directory on somebody's behalf, not +a model choosing to run something. Both are read-only, both are built here +rather than assembled from anything a model said, and the project directory is +configuration rather than input. It is still an exception to Manual mode's +"everything is shown to you before it happens", and it is written down in +CLAUDE.md next to the others. + +**Nothing here is trusted.** Filenames come off somebody else's machine and end +up inside a system prompt, so they are stripped of control characters, capped +in length, capped in number, and never interpreted. +""" + +from __future__ import annotations + +import asyncio +import logging +import re +import time +from dataclasses import dataclass, field + +from lembas.services.agent.base import ExecError, ExecRequest, Executor + +log = logging.getLogger(__name__) + +# How many paths are kept. Past this the index says it was truncated, which the +# rendering repeats to the model -- "there is nothing else here" and "I stopped +# looking" are different answers and it must not give the first for the second. +MAX_ENTRIES = 20_000 +# One path. Longer than any real one and shorter than an attack. +MAX_PATH = 400 +# How long a walk may take before it is abandoned. The index is a convenience; +# a chat must never sit waiting for one. +BUILD_TIMEOUT = 20.0 +# Output budget for the listing commands. Twenty thousand paths at forty +# characters is 800KB, so this has room and still refuses a runaway. +MAX_OUTPUT = 2 * 1024 * 1024 + +# How long a built index is reused, and how many are kept at once. A project +# directory changes under you -- the model writes files into it -- so this is +# short. `refresh` exists for when short is not short enough. +TTL = 300.0 +MAX_CACHED = 64 + +# How deep the SFTP fallback goes, and how many directories it will open. It is +# a round trip per directory, so an unbounded walk of somebody's home directory +# would take minutes and achieve nothing. +SFTP_MAX_DEPTH = 6 +SFTP_MAX_DIRS = 400 + +# Pruned from the `find` and SFTP paths. Not applied to `git ls-files`, which +# has already applied the repository's own rules and where a checked-in +# `vendor/` is checked in on purpose -- this project's own hash-pinned browser +# libraries live in one. +IGNORED = ( + ".git", + ".hg", + ".svn", + "node_modules", + "__pycache__", + ".venv", + "venv", + ".mypy_cache", + ".pytest_cache", + ".ruff_cache", + ".tox", + ".next", + ".nuxt", + ".gradle", + ".terraform", + "target", + "dist", + "build", + ".DS_Store", +) + +# Control characters, including the escape that would let a filename repaint +# the transcript it is quoted in. +_CONTROL = re.compile(r"[\x00-\x1f\x7f-\x9f]") + + +@dataclass(frozen=True) +class ProjectIndex: + """A snapshot of what was in a directory, and how it was found out.""" + + paths: tuple[str, ...] = () + total: int = 0 + truncated: bool = False + source: str = "" + built_at: float = field(default=0.0) + + @property + def ok(self) -> bool: + return bool(self.paths) + + +# --- Building ---------------------------------------------------------------- +def _clean(raw: str) -> str: + """One path, made safe to put in a prompt and in an attribute.""" + path = _CONTROL.sub("", raw.strip()).lstrip("./") + return path[:MAX_PATH] + + +def _collect(output: str) -> tuple[tuple[str, ...], int, bool]: + seen: set[str] = set() + paths: list[str] = [] + total = 0 + for line in output.splitlines(): + path = _clean(line) + if not path or path in seen: + continue + total += 1 + if len(paths) < MAX_ENTRIES: + seen.add(path) + paths.append(path) + paths.sort() + return tuple(paths), total, total > len(paths) + + +async def _from_git(executor: Executor, project_dir: str) -> ProjectIndex | None: + """Tracked and untracked files, minus whatever `.gitignore` excludes. + + `--exclude-standard` is what makes this worth trying first: the repository + already carries somebody's considered list of what is not part of the + project, and reproducing it by hand is how an index ends up ninety percent + build output. + """ + result = await executor.run( + ExecRequest( + command="git ls-files -c -o --exclude-standard 2>/dev/null", + cwd=project_dir, + timeout=BUILD_TIMEOUT, + max_bytes=MAX_OUTPUT, + ) + ) + if not result.ok or not result.output.strip(): + return None + paths, total, truncated = _collect(result.output) + if not paths: + return None + return ProjectIndex( + paths=paths, total=total, truncated=truncated or result.truncated, source="git" + ) + + +def _find_command() -> str: + prunes = " -o ".join(f"-name {name!r}" for name in IGNORED) + # -print rather than -print0: the output is read as text either way, and a + # filename containing a newline splits into two entries that resolve to + # nothing rather than into anything dangerous. + return f"find . \\( {prunes} \\) -prune -o -print 2>/dev/null" + + +async def _from_find(executor: Executor, project_dir: str) -> ProjectIndex | None: + result = await executor.run( + ExecRequest( + command=_find_command(), + cwd=project_dir, + timeout=BUILD_TIMEOUT, + max_bytes=MAX_OUTPUT, + ) + ) + if not result.output.strip(): + return None + paths, total, truncated = _collect(result.output) + if not paths: + return None + return ProjectIndex( + paths=paths, total=total, truncated=truncated or result.truncated, source="find" + ) + + +async def _from_sftp(executor: Executor, project_dir: str) -> ProjectIndex: + """The one that always works, and the one that is slow. + + Bounded twice over -- by depth and by how many directories it will open -- + because this is a network round trip per directory and an unbounded walk of + a home directory would take minutes to produce something unusable. + """ + found: list[str] = [] + opened = 0 + queue: list[tuple[str, int]] = [("", 0)] + + while queue and opened < SFTP_MAX_DIRS and len(found) < MAX_ENTRIES: + where, depth = queue.pop(0) + opened += 1 + try: + entries = await executor.scan_dir(where or project_dir) + except ExecError: + continue + for entry in entries: + if entry.name in IGNORED: + continue + path = f"{where}/{entry.name}" if where else entry.name + found.append(path + "/" if entry.is_dir else path) + if entry.is_dir and depth + 1 < SFTP_MAX_DEPTH: + queue.append((path, depth + 1)) + + paths, total, truncated = _collect("\n".join(found)) + return ProjectIndex( + paths=paths, + total=total, + truncated=truncated or bool(queue), + source="sftp", + ) + + +async def build(executor: Executor, project_dir: str) -> ProjectIndex: + """Walk the directory, by whichever means works first.""" + started = time.monotonic() + try: + for attempt in (_from_git, _from_find): + found = await attempt(executor, project_dir) + if found is not None: + break + else: + found = None + if found is None: + found = await _from_sftp(executor, project_dir) + except ExecError as exc: + log.info("could not index %s: %s", project_dir, exc.message) + return ProjectIndex(built_at=time.monotonic()) + + log.debug( + "indexed %s: %d paths by %s in %dms", + project_dir, + len(found.paths), + found.source, + int((time.monotonic() - started) * 1000), + ) + return ProjectIndex( + paths=found.paths, + total=found.total, + truncated=found.truncated, + source=found.source, + built_at=time.monotonic(), + ) + + +# --- The cache --------------------------------------------------------------- +# Keyed on the connection and the directory, not the chat: two chats on the same +# box in the same tree are looking at the same files, and indexing it twice +# would double the cost to prove it. +_CACHE: dict[tuple[str, str], ProjectIndex] = {} +_BUILDING: dict[tuple[str, str], asyncio.Task] = {} + + +def cached(profile_id: str, project_dir: str) -> ProjectIndex | None: + """What is already known, or None. Never does any work. + + `harness.context_variables` is synchronous and sits on the request path, so + it may only ever call this -- an SFTP round trip from there would block a + request while somebody's box thought about it. + """ + found = _CACHE.get((profile_id, project_dir)) + if found is None: + return None + if time.monotonic() - found.built_at > TTL: + _CACHE.pop((profile_id, project_dir), None) + return None + return found + + +async def ensure( + executor: Executor, profile_id: str, project_dir: str, *, refresh: bool = False +) -> ProjectIndex: + """The index, building it if there is not a fresh one already. + + Concurrent callers share one build. A reply and the `@` picker asking at + the same moment is the ordinary case, not a rare one, and two walks of the + same tree would be two of everything for one answer. + """ + key = (profile_id, project_dir) + if refresh: + _CACHE.pop(key, None) + elif (found := cached(profile_id, project_dir)) is not None: + return found + + if (running := _BUILDING.get(key)) is not None: + return await asyncio.shield(running) + + task = asyncio.create_task(build(executor, project_dir)) + _BUILDING[key] = task + try: + found = await task + finally: + _BUILDING.pop(key, None) + + _CACHE[key] = found + while len(_CACHE) > MAX_CACHED: + _CACHE.pop(next(iter(_CACHE))) + return found + + +# --- Rendering --------------------------------------------------------------- +# A tree that lists a thousand files is worse than no tree: it costs the window +# on every request forever and buries the four names that mattered. So the +# rendering has a character budget and elides what will not fit, saying how much +# it elided -- a directory shown as `src/vendor/ (412 files)` is a model being +# told where to look, which is the useful half of listing it. +INDENT = " " +# Below this a directory is never collapsed. Elision costs a line either way, so +# collapsing three files into "(3 files)" saves nothing and loses everything. +ALWAYS_SHOW = 4 + + +def _tree(paths: tuple[str, ...]) -> dict: + root: dict = {} + for path in paths: + node = root + parts = [part for part in path.rstrip("/").split("/") if part] + for part in parts[:-1]: + node = node.setdefault(part, {}) + if not isinstance(node, dict): # a file and a directory share a name + break + else: + if parts: + leaf = parts[-1] + if path.endswith("/"): + node.setdefault(leaf, {}) + else: + node.setdefault(leaf, None) + return root + + +def _files_under(node: dict) -> int: + total = 0 + for child in node.values(): + total += _files_under(child) if isinstance(child, dict) else 1 + return total + + +def _candidates(node: dict, prefix: str, depth: int, out: list) -> None: + """Every directory, with what collapsing it would save.""" + for name, child in node.items(): + if not isinstance(child, dict): + continue + path = f"{prefix}{name}/" + count = _files_under(child) + full = _cost(child, depth + 1) + collapsed = len(f" ({count} files)") + if count > ALWAYS_SHOW and full > collapsed: + out.append((depth, count, path, full - collapsed)) + _candidates(child, path, depth + 1, out) + + +def _cost(node: dict, depth: int) -> int: + """Roughly how many characters rendering this subtree in full would take.""" + total = 0 + for name, child in node.items(): + total += len(INDENT) * (depth + 1) + len(name) + 2 + if isinstance(child, dict): + total += _cost(child, depth + 1) + return total + + +def _plan(root: dict, budget: int) -> set[str]: + """Which directories to show as a count, so the rest fits. + + Deepest and largest first. Collapsing by saving alone would take `src/` + before `src/web/static/vendor/` -- it is bigger, because it *contains* it -- + and lose every name worth having to save one directory of hash-pinned + third-party files. Depth is the proxy for "further from what somebody was + looking for", and it is a good one. + """ + if _cost(root, 0) <= budget: + return set() + + candidates: list[tuple[int, int, str, int]] = [] + _candidates(root, "", 0, candidates) + candidates.sort(key=lambda item: (-item[0], -item[1])) + + chosen: dict[str, int] = {} + saved = 0 + total = _cost(root, 0) + for _depth, _count, path, saving in candidates: + if total - saved <= budget: + break + # A directory inside one already collapsed is not rendered at all, so + # collapsing it saves nothing. + if any(path.startswith(done) for done in chosen): + continue + # And a directory *containing* one already collapsed subsumes it. Its + # own saving is measured against the full subtree, so the descendant's + # has to come back off or the two are counted twice -- which stopped + # the loop early believing it had made room it had not. + for inside in [done for done in chosen if done.startswith(path)]: + saved -= chosen.pop(inside) + chosen[path] = saving + saved += saving + return set(chosen) + + +def _lines( + node: dict, prefix: str, depth: int, collapsed: set[str], budget: list[int] +) -> list[str]: + out: list[str] = [] + # Files before directories at each level: the shallow names are the ones + # somebody would recognise, and if the budget runs out mid-tree they are + # the ones worth having spent it on. + files = sorted(name for name, child in node.items() if not isinstance(child, dict)) + folders = sorted(name for name, child in node.items() if isinstance(child, dict)) + + for position, name in enumerate(files): + line = f"{INDENT * depth}{name}" + if budget[0] < len(line) + 1: + out.append(f"{INDENT * depth}… {len(files) - position} more files") + budget[0] = 0 + return out + budget[0] -= len(line) + 1 + out.append(line) + + for name in folders: + child = node[name] + path = f"{prefix}{name}/" + header = f"{INDENT * depth}{name}/" + if path in collapsed: + line = f"{header} ({_files_under(child)} files)" + budget[0] -= len(line) + 1 + out.append(line) + continue + if budget[0] < len(header) + 1: + return out + budget[0] -= len(header) + 1 + out.append(header) + out.extend(_lines(child, path, depth + 1, collapsed, budget)) + return out + + +def render(index: ProjectIndex, budget: int) -> str: + """The listing as the model sees it, inside `budget` characters. + + Returns "" when there is nothing to say, so the fragment carrying it can + vanish entirely rather than appear as an empty heading -- which is what + `Fragment.requires` is for. + """ + if not index.ok or budget <= 0: + return "" + + root = _tree(index.paths) + collapsed = _plan(root, budget) + # The plan has already made it fit, so this is a backstop rather than the + # mechanism -- with enough slack that an estimate a little off does not + # truncate a listing that was fine. What it is really for is the one shape + # collapsing cannot help with: five thousand files directly in the root, + # where there is no directory to fold them into. + remaining = [int(budget * 1.5) + 200] + lines = _lines(root, "", 0, collapsed, remaining) + if not lines: + return "" + + note = "" + if index.truncated: + note = ( + f"\n\nThere are more than {len(index.paths)} entries here; this is the " + "first of them, so treat it as a sample rather than the whole tree." + ) + elif collapsed: + note = ( + "\n\nDirectories shown with a count were left unopened to save room. " + "Use `file_list` to look inside one." + ) + return "\n".join(lines) + note + + +def forget(profile_id: str) -> int: + """Drop everything indexed through one connection. + + Called when a profile is deleted, disabled or has its host key forgotten -- + the same moments that close its terminals. Keeping a listing of a machine + somebody has just revoked would be a small leak of exactly the kind the + rest of this module is careful about. + """ + doomed = [key for key in _CACHE if key[0] == profile_id] + for key in doomed: + _CACHE.pop(key, None) + return len(doomed) + + +def clear() -> None: + _CACHE.clear() + + +__all__ = [ + "MAX_ENTRIES", + "ProjectIndex", + "build", + "cached", + "clear", + "ensure", + "forget", + "render", +] diff --git a/src/lembas/services/agent/ssh.py b/src/lembas/services/agent/ssh.py index 8be3198..132dbee 100644 --- a/src/lembas/services/agent/ssh.py +++ b/src/lembas/services/agent/ssh.py @@ -385,6 +385,7 @@ async def check(spec: dict[str, Any], project_dir: str = "") -> dict[str, Any]: __all__ = [ "INSTALL_HINT", + "MAX_ENTRIES", "MAX_READ_BYTES", "SshExecutor", "available", diff --git a/src/lembas/services/chat.py b/src/lembas/services/chat.py index 2373b12..4b8a5d8 100644 --- a/src/lembas/services/chat.py +++ b/src/lembas/services/chat.py @@ -89,14 +89,30 @@ def document_context(message: Message) -> str: if not attachment.extracted_text.strip(): continue note = " (truncated)" if attachment.truncated else "" + # Where it came from, when there is a where. A model handed `main.py` + # cannot tell which of four it is looking at, and cannot name the file + # back when asked to change something -- so a file read off a machine + # says which machine and which path. Quotes are stripped rather than + # escaped: these are attribute values in a tag the model reads, and a + # path containing one would otherwise close it early. + where = "" + if attachment.source_path: + where += f' path="{_attr(attachment.source_path)}"' + if attachment.source_label: + where += f' from="{_attr(attachment.source_label)}"' blocks.append( - f'\n' + f'\n' f"{attachment.extracted_text.strip()}\n" f"" ) return "\n\n".join(blocks) +def _attr(value: str) -> str: + """A value safe to sit inside the double quotes of a tag we are writing.""" + return value.replace('"', "").replace("<", "").replace(">", "").replace("\n", " ") + + def message_payload(message: Message, *, vision: bool) -> dict[str, Any]: """One history entry in the shape the endpoint expects. diff --git a/src/lembas/services/files.py b/src/lembas/services/files.py index 26f500b..8850117 100644 --- a/src/lembas/services/files.py +++ b/src/lembas/services/files.py @@ -334,12 +334,20 @@ def store_text( text: str, truncated: bool = False, source_note: str = "", + source_path: str = "", + source_label: str = "", ) -> Attachment: """Attach text that did not arrive as a file -- a fetched web page. Written to disk like any other attachment so it can be downloaded and so there is one cleanup path, rather than a second kind of attachment that exists only in the database. + + `source_note` leads the *text*; `source_path` and `source_label` are + columns. The two are not the same thing and both are wanted: the note is + prose a model reads inside the document, and the columns become attributes + on the tag around it, which is what a reader sees on the chip and what + survives if the text is later truncated away from its own first line. """ body = text[:MAX_EXTRACTED_CHARS] payload = body.encode("utf-8") @@ -359,6 +367,8 @@ def store_text( # can see where an attachment called "Some Page.txt" came from. extracted_text=f"Source: {source_note}\n\n{body}" if source_note else body, truncated=truncated, + source_path=source_path[:1000], + source_label=source_label[:200], ) db.add(attachment) db.commit() diff --git a/src/lembas/services/generation.py b/src/lembas/services/generation.py index 72eb0db..fdf18bc 100644 --- a/src/lembas/services/generation.py +++ b/src/lembas/services/generation.py @@ -26,7 +26,7 @@ from datetime import UTC, datetime, timedelta from sqlalchemy import select -from lembas.db.models import ROLE_ASSISTANT, ROLE_USER, Chat, Message, User +from lembas.db.models import KIND_AGENT, ROLE_ASSISTANT, ROLE_USER, Chat, Message, User from lembas.db.session import session_scope from lembas.services import chat as chat_service from lembas.services import compaction as compaction_service @@ -294,6 +294,12 @@ async def _run(generation: Generation) -> None: # summarisation in front of it would break exactly that. await _maybe_compact(generation) + # Before the session opens, for the same reason compaction is: the + # listing is an SSH round trip, and holding a database session across + # one to save opening a second is the wrong trade. `build_request` + # below reads whatever this left in the cache and never fetches. + await _warm_index(generation) + with session_scope() as db: chat = db.get(Chat, generation.chat_id) message = db.get(Message, generation.message_id) @@ -524,6 +530,55 @@ async def _run(generation: Generation) -> None: generation.touch() +# How long a reply will wait for a directory listing before starting without +# one. Short on purpose: the listing is a convenience and the reply is the +# thing somebody is waiting for. A walk that outruns this keeps going in the +# background and the next turn has it. +INDEX_WAIT = 6.0 + + +async def _warm_index(generation: Generation) -> None: + """Build this chat's project listing, or leave whatever is cached. + + Never raises and never blocks for long. `harness` reads the cache + synchronously while assembling the system message, so something has to fill + it, and this is the one place in a reply's life that is both asynchronous + and already doing network work. + + The first reply in a brand-new chat on a big tree may start before the walk + finishes. That is deliberate: the fragment carrying the listing vanishes + when it is empty rather than appearing as a heading with nothing under it, + and by the following turn it is there. + """ + from lembas.services import settings_store + from lembas.services.agent import index as index_service + from lembas.services.agent import session as agent_session + + try: + with session_scope() as db: + if not settings_store.agents(db).get("index_enabled"): + return + chat = db.get(Chat, generation.chat_id) + if chat is None or chat.kind != KIND_AGENT: + return + owner = db.get(User, chat.user_id) + context = agent_session.resolve(db, chat, owner) + profile_id = chat.ssh_profile_id or "" + if context is None or not profile_id: + return + if index_service.cached(profile_id, context.project_dir) is not None: + return + + await asyncio.wait_for( + index_service.ensure(context.executor(), profile_id, context.project_dir), + timeout=INDEX_WAIT, + ) + except TimeoutError: + log.debug("index for chat %s outran its wait; carrying on", generation.chat_id) + except Exception as exc: # noqa: BLE001 - a missing listing is not a failed reply + log.info("could not warm the index for chat %s: %s", generation.chat_id, exc) + + async def _maybe_compact(generation: Generation) -> None: """Summarise the earlier turns if the window is about to be full. diff --git a/src/lembas/services/harness.py b/src/lembas/services/harness.py index 51d548b..c5e26b2 100644 --- a/src/lembas/services/harness.py +++ b/src/lembas/services/harness.py @@ -135,6 +135,7 @@ def context_variables( "agent_dir": "", "agent_mode": "", "agent_rewound": "", + "project_files": "", } if chat is not None: @@ -161,6 +162,8 @@ def context_variables( def _agent_values(db: DBSession, chat, user) -> dict[str, str]: """What an agent chat's harness needs to say about where it is.""" + from lembas.services import settings_store + from lembas.services.agent import index as index_service from lembas.services.agent import policy from lembas.services.agent import session as agent_session @@ -178,9 +181,36 @@ def _agent_values(db: DBSession, chat, user) -> dict[str, str]: "agent_mode": policy.MODE_GUIDANCE.get(context.mode, ""), "agent_rewound": rewound, "max_rounds": str(context.limits.steps), + "project_files": _project_files(db, chat, context, settings_store, index_service), } +def _project_files(db: DBSession, chat, context, settings_store, index_service) -> str: + """The directory listing, *read from cache and never fetched*. + + This whole module runs synchronously on the request path, so an SFTP round + trip here would hold a request open while somebody's box thought about it. + The build happens in the generation setup, which is async and already doing + network work; here we take whatever it left behind. + + A chat whose very first reply outruns its first index simply has no listing + that turn -- the fragment's `requires` makes it vanish rather than appear as + an empty heading, and the next turn has it. + """ + agents = settings_store.agents(db) + if not agents.get("index_enabled"): + return "" + budget = int(agents.get("index_chars") or 0) + if budget <= 0: + return "" + + profile_id = getattr(chat, "ssh_profile_id", "") or "" + found = index_service.cached(profile_id, context.project_dir) + if found is None: + return "" + return index_service.render(found, budget) + + def limit_for(db: DBSession) -> int: """The ceiling on the assembled block.""" stored = settings_store.get(db, "max_harness_chars", key=settings_store.PROMPTS) diff --git a/src/lembas/services/prompts.py b/src/lembas/services/prompts.py index d6737be..13cf361 100644 --- a/src/lembas/services/prompts.py +++ b/src/lembas/services/prompts.py @@ -155,6 +155,14 @@ VARIABLES: tuple[Variable, ...] = ( "When an agent chat was last edited or regenerated. Empty otherwise, " "which is what keeps the note about it out of every other reply.", ), + Variable( + "project_files", + "Project files", + "What is in the project directory, as an indented tree with large " + "directories shown as a count. Empty until the first listing has been " + "built, when the feature is off, or when the directory could not be " + "read -- and the section it lives in disappears with it.", + ), Variable( "memories", "Memories", @@ -805,6 +813,33 @@ BUILTIN: tuple[Fragment, ...] = ( "not look for another way round it." ), ), + Fragment( + key="tool.project_files", + label="What is in the project directory", + group=GROUP_CONTEXT, + order=325, + families=("agent",), + requires=("project_files",), + variables=("project_files", "agent_dir"), + hint="A listing of the project directory, so the first two rounds of a " + "reply are not spent finding out what is in it. Large directories are " + "shown as a count rather than expanded, and the budget for the whole " + "thing is set under Admin -> Agents. Clearing this box switches the " + "listing off in the prompt while leaving it available to the file " + "picker.", + default=( + "### Files in {{agent_dir}}\n" + "\n" + "```\n" + "{{project_files}}\n" + "```\n" + "\n" + "A snapshot from when this reply started, and not necessarily " + "complete. It is a map, not an authority: check a path before " + "relying on it, and do not conclude a file is absent because it is " + "not listed here." + ), + ), Fragment( key="tool.agent_rewound", label="After a rewind", diff --git a/src/lembas/services/settings_store.py b/src/lembas/services/settings_store.py index e3a1369..386cdf8 100644 --- a/src/lembas/services/settings_store.py +++ b/src/lembas/services/settings_store.py @@ -88,6 +88,14 @@ def _agents_defaults() -> dict[str, Any]: # SSH connection held open, so this is a real resource, not a scruple. "terminal_max_sessions": 20, "terminal_max_per_user": 3, + # A listing of the project directory, put in front of the model so the + # first rounds of a reply are not spent discovering what is there. It + # costs its budget on *every* request in an agent chat, forever, which + # is why it is a switch and a number rather than a constant. + "index_enabled": True, + # Characters. Clamped on read: a huge value here would quietly spend + # somebody's whole context window on filenames. + "index_chars": 2000, } @@ -244,4 +252,8 @@ def agents(db: DBSession) -> dict[str, Any]: max(int(values.get("terminal_max_sessions") or 0), 1), 500 ) values["terminal_max_per_user"] = min(max(int(values.get("terminal_max_per_user") or 0), 1), 50) + # Zero is meaningful here and is not clamped away: it means "index the + # directory for the file picker, but put none of it in the prompt", which + # is a reasonable thing to want and has no other way of being said. + values["index_chars"] = min(max(int(values.get("index_chars") or 0), 0), 20_000) return values diff --git a/src/lembas/web/static/css/chat.css b/src/lembas/web/static/css/chat.css index 9162fc5..700e744 100644 --- a/src/lembas/web/static/css/chat.css +++ b/src/lembas/web/static/css/chat.css @@ -667,7 +667,12 @@ border-top: 1px solid var(--border); background: var(--bg); } -.composer__inner { max-width: var(--thread-max-width); margin: 0 auto; } +/* position: relative anchors the `@` and `/` menu to the box. */ +.composer__inner { + position: relative; + max-width: var(--thread-max-width); + margin: 0 auto; +} /* A column: chips, then the text across the full width, then one toolbar row. @@ -765,6 +770,58 @@ color: var(--ink-faint); text-align: center; } +.composer__hint-link { + border: 0; + padding: 0; + background: none; + font: inherit; + color: var(--ink-muted); + text-decoration: underline dotted; + text-underline-offset: 2px; + cursor: pointer; +} +.composer__hint-link:hover { color: var(--accent); } + +/* + The menu `@` and `/` open. + + Above the composer, not below it: the composer is already at the bottom of + the window, so anything dropping down would be off screen -- the same reason + the attach menu is `picker--up`. Anchored to .composer__inner so it lines up + with the box rather than with the caret; a caret-following menu is nicer and + needs measuring text in a textarea, which cannot be done without a hidden + mirror element. +*/ +.composer-menu { + position: absolute; + bottom: calc(100% + var(--sp-2)); + left: 0; + right: 0; + z-index: var(--z-dropdown); + max-height: min(20rem, 45vh); + overflow-y: auto; + scrollbar-width: thin; + border: 1px solid var(--border); + border-radius: var(--radius-lg); + background: var(--surface-raised); + box-shadow: var(--shadow-lg); +} + +.picker__group { + margin: 0; + padding: var(--sp-2) var(--sp-3) var(--sp-1); + font-size: var(--text-xs); + font-weight: 600; + letter-spacing: 0.04em; + text-transform: uppercase; + color: var(--ink-faint); +} + +/* The help and usage sheets. */ +.sheet { width: 100%; border-collapse: collapse; font-size: var(--text-sm); } +.sheet td { padding: var(--sp-1) var(--sp-2); vertical-align: top; } +.sheet td:first-child { white-space: nowrap; color: var(--ink-muted); width: 1%; } +.sheet tr + tr td { border-top: 1px solid var(--border); } /* --- Folders -------------------------------------------------------------- */ .folder__row { padding-right: var(--sp-1); } diff --git a/src/lembas/web/static/js/commands.js b/src/lembas/web/static/js/commands.js new file mode 100644 index 0000000..cbf9794 --- /dev/null +++ b/src/lembas/web/static/js/commands.js @@ -0,0 +1,385 @@ +/* + Slash commands, and the keyboard shortcuts that do the same jobs. + + Both live here, in one table, so `/help` cannot describe a shortcut that no + longer exists. Every entry does something that was already possible by + clicking -- none of this is new server behaviour except `/usage`, which is a + question the interface could not previously answer at all. + + The rule that matters: a message that merely *starts* with a slash must still + send. `//` escapes, an unrecognised command is left alone and posted as text, + and only an exact match against this table is intercepted. Silently eating + somebody's message is a far worse failure than an unknown command. + + Shortcuts are Alt-based rather than Ctrl+Shift: the browser owns + Ctrl+Shift+T, N and W and will not give them up. They are matched on + `event.code`, which is the physical key, so a Dvorak or a Slovak layout gets + the same shortcuts rather than whichever letters happen to sit there. +*/ +(function () { + "use strict"; + + function el(selector) { return document.querySelector(selector); } + function chat() { + var box = el(".composer"); + return (box && box.dataset.chatId) || ""; + } + function isAgent() { return !!el('[name="agent_mode"]'); } + function post(url, options) { + return fetch(url, Object.assign({ method: "POST", credentials: "same-origin" }, options || {})); + } + function note(message, kind) { + if (window.lembas && window.lembas.notify) { + window.lembas.notify(message, { kind: kind || "info" }); + } + } + + /* --- Shortcuts ---------------------------------------------------------- */ + var SHORTCUTS = [ + { keys: "Ctrl/⌘ + K", what: "Open the command menu" }, + { keys: "Alt + 1 … 4", what: "Manual, Edit, Auto, Plan" }, + { keys: "Alt + T", what: "Terminal" }, + { keys: "Alt + I", what: "Inspector" }, + { keys: "Alt + B", what: "Sidebar" }, + { keys: "Alt + N", what: "New chat" }, + { keys: "↑ in an empty box", what: "Edit your last message" }, + { keys: "Esc", what: "Close what is open, or stop the reply" }, + { keys: "Ctrl + Shift + C / V", what: "Copy and paste inside the terminal" } + ]; + + /* --- The table ---------------------------------------------------------- */ + var COMMANDS = [ + { + name: "help", + summary: "Commands and keyboard shortcuts", + run: function () { helpSheet(); } + }, + { + name: "usage", + summary: "Tokens and context used by this chat", + when: function () { return !!chat(); }, + run: function () { + fetch("/api/chats/" + chat() + "/usage", { credentials: "same-origin" }) + .then(function (r) { return r.text(); }) + .then(function (html) { sheet("Usage", html); }) + .catch(function () { note("Could not read this chat's usage.", "error"); }); + } + }, + { + name: "compact", + summary: "Summarise the earlier turns so they stop costing context", + when: function () { return !!chat() && !!el("#thread .msg"); }, + run: function () { + window.lembas.confirm( + "Summarise everything before the last reply? The messages stay in the " + + "transcript; they just stop being sent to the model.", + { title: "Compact this chat", label: "Compact" } + ).then(function (yes) { + if (!yes) return; + post("/api/chats/" + chat() + "/compact") + .then(function (r) { return r.ok ? r.text() : Promise.reject(r); }) + .then(function (html) { + el("#thread").innerHTML = html; + if (window.htmx) window.htmx.process(el("#thread")); + }) + .catch(function (r) { + if (r && r.json) r.json().then(function (body) { note(body.detail, "error"); }); + else note("Could not compact this chat.", "error"); + }); + }); + } + }, + { + name: "mode", + summary: "Approval mode: manual, edit, auto or plan", + argument: "manual | edit | auto | plan", + when: isAgent, + run: function (rest) { + var select = el('[name="agent_mode"]'); + var wanted = (rest || "").trim().toLowerCase(); + if (!wanted) return note("Modes: manual, edit, auto, plan."); + var found = Array.prototype.find.call(select.options, function (option) { + return option.value === wanted; + }); + if (!found) return note("“" + wanted + "” is not a mode.", "error"); + select.value = wanted; + select.dispatchEvent(new Event("change", { bubbles: true })); + note("Mode set to " + found.textContent.trim() + "."); + } + }, + { + name: "title", + summary: "Rename this chat", + argument: "the new title", + when: function () { return !!chat(); }, + run: function (rest) { + var wanted = (rest || "").trim(); + if (!wanted) return note("Give it a title: /title Something."); + var body = new FormData(); + body.append("title", wanted); + fetch("/api/chats/" + chat(), { method: "PATCH", body: body, credentials: "same-origin" }) + .then(function () { + var heading = el("#chat-title"); + // textContent, never innerHTML: this is text somebody typed. + if (heading) heading.textContent = wanted; + note("Renamed."); + }); + } + }, + { + name: "terminal", + summary: "Show or hide the terminal", + when: function () { return !!el("#terminal"); }, + run: function () { toggle("#terminal", "side"); } + }, + { + name: "inspector", + summary: "Show or hide the request inspector", + when: function () { return !!el("#inspector"); }, + run: function () { toggle("#inspector", "side"); } + }, + { name: "sidebar", summary: "Show or hide the sidebar", run: function () { toggle("#sidebar"); } }, + { + name: "theme", + summary: "Switch theme", + argument: "moria | shire", + run: function (rest) { + var wanted = (rest || "").trim().toLowerCase(); + if (wanted === "moria" || wanted === "shire") window.lembas.applyTheme(wanted); + else window.lembas.toggleTheme(); + } + }, + { name: "new", summary: "Start a new chat", run: function () { window.location = "/chat"; } }, + { + name: "temp", + summary: "Start a temporary chat, gone after a day", + run: function () { window.location = "/chat?temporary=1"; } + }, + { + name: "stop", + summary: "Stop the reply being written", + run: function () { + var button = el('[data-composer-action="stop"]'); + if (button) button.click(); + else note("Nothing is being written."); + } + }, + { name: "knowledge", summary: "Your library", run: go("/library/knowledge") }, + { name: "notes", summary: "Notes the model has written", run: go("/library/notes") }, + { name: "skills", summary: "Saved procedures", run: go("/library/skills") }, + { name: "connections", summary: "Your SSH connections", run: go("/agents") } + ]; + + function go(url) { + return function () { window.location = url; }; + } + + function toggle(selector, group) { + var panel = el(selector); + if (panel && window.lembas.setPanel) { + window.lembas.setPanel(selector, panel.hasAttribute("hidden"), group); + } + } + + function available() { + return COMMANDS.filter(function (command) { return !command.when || command.when(); }); + } + + /* --- What the composer calls -------------------------------------------- */ + function list(query) { + var needle = (query || "").toLowerCase(); + var matches = available().filter(function (command) { + return command.name.indexOf(needle) === 0; + }); + + var wrap = document.createElement("div"); + if (!matches.length) { + var empty = document.createElement("p"); + empty.className = "muted text-sm"; + empty.style.padding = "var(--sp-3)"; + empty.textContent = "No command called “" + query + "”. It will be sent as a message."; + wrap.appendChild(empty); + return wrap; + } + + var group = document.createElement("p"); + group.className = "picker__group"; + group.textContent = "Commands"; + wrap.appendChild(group); + + var items = document.createElement("ul"); + items.className = "picker__list"; + matches.forEach(function (command) { + var row = document.createElement("li"); + var button = document.createElement("button"); + button.type = "button"; + button.className = "picker__option"; + button.dataset.command = command.name; + + var body = document.createElement("span"); + body.className = "picker__option-body"; + var name = document.createElement("span"); + name.className = "picker__option-name"; + name.textContent = "/" + command.name + (command.argument ? " " + command.argument : ""); + var summary = document.createElement("span"); + summary.className = "picker__option-note"; + summary.textContent = command.summary; + body.appendChild(name); + body.appendChild(summary); + button.appendChild(body); + row.appendChild(button); + items.appendChild(row); + }); + wrap.appendChild(items); + return wrap; + } + + /* + A command, or null -- and null is the important half. + + Anything not matching exactly is left for the composer to send as an + ordinary message, and `//` strips one slash on the way. A chat application + that swallows a message because it began with a slash has done something + much worse than failing to recognise a command. + */ + function find(value) { + if (value[0] !== "/" || value[1] === "/") return null; + var match = /^\/([a-z]+)(?:\s+([\s\S]*))?$/.exec(value.trim()); + if (!match) return null; + var found = available().find(function (command) { return command.name === match[1]; }); + return found ? { name: found.name, rest: match[2] || "" } : null; + } + + function run(name, rest) { + var found = available().find(function (command) { return command.name === name; }); + if (found) found.run(rest || ""); + } + + /* --- Sheets ------------------------------------------------------------- */ + function sheet(title, html) { + var dialog = document.createElement("dialog"); + dialog.className = "dialog dialog--wide"; + var form = document.createElement("div"); + form.className = "dialog__form"; + var heading = document.createElement("h2"); + heading.className = "dialog__title"; + heading.textContent = title; + var body = document.createElement("div"); + // Server-rendered and already escaped there; nothing user-typed reaches + // this path as markup. + body.innerHTML = html; + var actions = document.createElement("div"); + actions.className = "dialog__actions"; + var close = document.createElement("button"); + close.className = "btn"; + close.type = "button"; + close.textContent = "Close"; + actions.appendChild(close); + form.appendChild(heading); + form.appendChild(body); + form.appendChild(actions); + dialog.appendChild(form); + document.body.appendChild(dialog); + + function finish() { + dialog.close(); + setTimeout(function () { dialog.remove(); }, 200); + } + close.addEventListener("click", finish); + dialog.addEventListener("cancel", function (event) { event.preventDefault(); finish(); }); + dialog.addEventListener("click", function (event) { if (event.target === dialog) finish(); }); + dialog.showModal(); + } + + function helpSheet() { + var rows = available().map(function (command) { + return ( + "/" + command.name + + (command.argument ? " " + escapeText(command.argument) : "") + + "" + escapeText(command.summary) + "" + ); + }); + var keys = SHORTCUTS.map(function (shortcut) { + return ( + "" + escapeText(shortcut.keys) + "" + + escapeText(shortcut.what) + "" + ); + }); + sheet( + "Commands and shortcuts", + "" + rows.join("") + "
" + + "

Keyboard

" + + "" + keys.join("") + "
" + + "

A message that starts with a slash but is not a " + + "command is sent as written. Type // to start one " + + "with a literal slash.

" + ); + } + + function escapeText(value) { + var holder = document.createElement("span"); + holder.textContent = value; + return holder.innerHTML; + } + + /* --- Keyboard ----------------------------------------------------------- */ + var MODES = ["manual", "edit", "auto", "plan"]; + + document.addEventListener("keydown", function (event) { + if (event.isComposing) return; + + /* Never inside the terminal: every keystroke there belongs to the shell, + and a shortcut that steals one is a shortcut that breaks vim. */ + if (event.target.closest && event.target.closest("#terminal")) return; + + if ((event.ctrlKey || event.metaKey) && event.code === "KeyK") { + event.preventDefault(); + var input = document.querySelector("[data-composer-input]"); + if (!input) return; + input.focus(); + input.value = "/"; + input.dispatchEvent(new Event("input", { bubbles: true })); + return; + } + + if (!event.altKey || event.ctrlKey || event.metaKey) return; + + if (event.code === "KeyT" && el("#terminal")) { + event.preventDefault(); + return toggle("#terminal", "side"); + } + if (event.code === "KeyI" && el("#inspector")) { + event.preventDefault(); + return toggle("#inspector", "side"); + } + if (event.code === "KeyB") { + event.preventDefault(); + return toggle("#sidebar"); + } + if (event.code === "KeyN") { + event.preventDefault(); + window.location = "/chat"; + return; + } + var digit = ["Digit1", "Digit2", "Digit3", "Digit4"].indexOf(event.code); + if (digit !== -1 && isAgent()) { + event.preventDefault(); + run("mode", MODES[digit]); + } + }); + + /* Up-arrow in an empty box edits your last turn, the way a shell recalls the + last command. Only when the box is empty, so it never eats a cursor key + somebody was using to move around what they had written. */ + document.addEventListener("keydown", function (event) { + if (event.key !== "ArrowUp" || event.shiftKey || event.altKey) return; + var input = event.target.closest && event.target.closest("[data-composer-input]"); + if (!input || input.value !== "") return; + var edits = document.querySelectorAll("#thread .msg--user [data-edit-message]"); + if (!edits.length) return; + event.preventDefault(); + edits[edits.length - 1].click(); + }); + + window.lembasCommands = { list: list, find: find, run: run, help: helpSheet }; +})(); diff --git a/src/lembas/web/static/js/composer.js b/src/lembas/web/static/js/composer.js new file mode 100644 index 0000000..9875d37 --- /dev/null +++ b/src/lembas/web/static/js/composer.js @@ -0,0 +1,326 @@ +/* + Typing affordances in the composer: `@` to attach something by name, and + (from the commands section below) `/` to run something instead of sending. + + Both are the same shape -- a token at the caret opens a menu, the menu + filters as you type, and choosing replaces the token -- so they share one + menu and one keyboard handler rather than fighting over the composer. + + Three things here are not obvious. + + The keydown listener is registered with `capture: true`. app.js already has a + document-level Enter handler that submits the form, and listeners on the same + element in the same phase fire in registration order -- app.js loads first, + so a bubble-phase listener here would never get to say "that Enter chose a + menu item, it did not send the message". + + Nothing is inserted into the composer as HTML. Every name in the menu came + off somebody's filesystem or out of their library. + + A chip is a chip. Choosing a file posts to a route that returns the same + attachment chip an upload returns, so the composer learns nothing new and the + remove button, the hidden file_ids input and `claim()` on send all work + already. +*/ +(function () { + "use strict"; + + var menu = null; + var list = null; + var open = false; + var kind = ""; + var pending = null; + var active = -1; + + /* commands.js loads first and defines these. Guarded anyway so that a page + which does not carry it -- or one where it failed to parse -- still gets + `@`, and a message beginning with a slash is simply sent. */ + function commands() { + return window.lembasCommands || { + list: function () { return document.createElement("div"); }, + find: function () { return null; }, + run: function () {} + }; + } + + function commandList(query) { return commands().list(query); } + function commandIn(value) { return commands().find(value); } + function runCommand(name, rest) { commands().run(name, rest); } + + /* --- Where we are ------------------------------------------------------- */ + function composer() { + return document.querySelector("[data-composer-input]"); + } + + function context() { + var box = document.querySelector(".composer"); + var picker = document.querySelector('select[name="ssh_profile_id"]'); + var dir = document.querySelector("[data-dir-value]"); + return { + chatId: (box && box.dataset.chatId) || "", + /* A chat under way carries its connection on the composer; a new one is + still choosing it, so the select and the hidden field are the truth. */ + profileId: picker ? picker.value : (box && box.dataset.profileId) || "", + projectDir: dir ? dir.value : (box && box.dataset.projectDir) || "" + }; + } + + /* + The token being typed, or null. + + `@` is claimed anywhere it follows whitespace, so `see @src/main.py` works + mid-sentence, but not inside an email address -- `a@b` is not a mention and + treating it as one would open a menu every time somebody typed one. + */ + function tokenAt(input) { + var value = input.value; + var caret = input.selectionStart; + if (caret !== input.selectionEnd) return null; + + var before = value.slice(0, caret); + var at = before.lastIndexOf("@"); + if (at !== -1 && (at === 0 || /\s/.test(before[at - 1]))) { + var query = before.slice(at + 1); + if (!/\s/.test(query)) return { kind: "@", query: query, start: at, end: caret }; + } + /* A slash command is only ever the first thing in the box. Anywhere else a + slash is a path, a date or a fraction. */ + if (before[0] === "/" && before[1] !== "/") { + var word = before.slice(1); + if (!/\s/.test(word) && caret === value.length) { + return { kind: "/", query: word, start: 0, end: caret }; + } + } + return null; + } + + /* --- The menu ----------------------------------------------------------- */ + function build() { + if (menu) return; + var host = document.querySelector(".composer__inner"); + if (!host) return; + menu = document.createElement("div"); + menu.className = "composer-menu"; + menu.setAttribute("role", "listbox"); + menu.hidden = true; + list = document.createElement("div"); + menu.appendChild(list); + host.insertBefore(menu, host.firstChild); + + menu.addEventListener("mousedown", function (event) { + /* Before the composer loses focus, or the caret position the choice is + about to be written at is already gone. */ + event.preventDefault(); + }); + menu.addEventListener("click", function (event) { + var option = event.target.closest("[data-mention-token], [data-command]"); + if (option) choose(option); + }); + } + + function show() { + build(); + if (!menu) return; + menu.hidden = false; + open = true; + } + + function hide() { + if (!menu) return; + menu.hidden = true; + open = false; + kind = ""; + active = -1; + } + + function options() { + return menu ? Array.prototype.slice.call( + menu.querySelectorAll("[data-mention-token], [data-command]") + ) : []; + } + + function highlight(index) { + var all = options(); + if (!all.length) return; + active = (index + all.length) % all.length; + all.forEach(function (option, position) { + option.classList.toggle("is-selected", position === active); + }); + if (all[active].scrollIntoView) all[active].scrollIntoView({ block: "nearest" }); + } + + /* --- Filling it --------------------------------------------------------- */ + function loadMentions(query) { + var where = context(); + var url = + "/api/files/mention-picker?q=" + encodeURIComponent(query) + + "&chat_id=" + encodeURIComponent(where.chatId) + + "&profile_id=" + encodeURIComponent(where.profileId) + + "&project_dir=" + encodeURIComponent(where.projectDir); + + fetch(url, { credentials: "same-origin" }) + .then(function (response) { return response.text(); }) + .then(function (html) { + if (kind !== "@") return; + list.innerHTML = html; + show(); + highlight(0); + }) + .catch(function () { hide(); }); + } + + function refresh() { + var input = composer(); + if (!input) return; + var token = tokenAt(input); + if (!token) return hide(); + + kind = token.kind; + if (token.kind === "/") { + list.innerHTML = ""; + list.appendChild(commandList(token.query)); + show(); + highlight(0); + return; + } + + clearTimeout(pending); + pending = setTimeout(function () { loadMentions(token.query); }, 150); + } + + /* --- Choosing ----------------------------------------------------------- */ + function replaceToken(text) { + var input = composer(); + var token = tokenAt(input); + if (!input || !token) return; + var head = input.value.slice(0, token.start); + var tail = input.value.slice(token.end); + var written = (token.kind === "@" ? "@" : "/") + text + " "; + input.value = head + written + tail; + var caret = head.length + written.length; + input.setSelectionRange(caret, caret); + if (window.lembas && window.lembas.autosize) window.lembas.autosize(input); + input.focus(); + } + + function choose(option) { + if (option.dataset.command) { + hide(); + runCommand(option.dataset.command, ""); + return; + } + + var where = context(); + /* The reference stays in the sentence being written *and* the contents + come along as a chip. The first is what makes "change the thing in + @main.py" read as a sentence; the second is what stops a small model + having to spend a round fetching it. */ + replaceToken(option.dataset.mentionToken); + hide(); + + var body = new FormData(); + body.append("chat_id", where.chatId); + if (option.dataset.mentionFile) { + body.append("profile_id", where.profileId); + body.append("path", option.dataset.mentionFile); + attach("/api/files/from-project", body); + } else if (option.dataset.mentionKnowledge) { + body.append("document_id", option.dataset.mentionKnowledge); + attach("/api/files/from-knowledge", body); + } + } + + function attach(url, body) { + var target = document.getElementById("attachments"); + if (!target) return; + fetch(url, { method: "POST", body: body, credentials: "same-origin" }) + .then(function (response) { return response.text(); }) + .then(function (html) { + target.insertAdjacentHTML("beforeend", html); + // The chip's remove button is htmx-driven and inert until announced. + if (window.htmx) window.htmx.process(target.lastElementChild); + }) + .catch(function () { + if (window.lembas) window.lembas.notify("Could not attach that.", { kind: "error" }); + }); + } + + /* --- Keys --------------------------------------------------------------- */ + document.addEventListener( + "keydown", + function (event) { + var input = event.target.closest && event.target.closest("[data-composer-input]"); + if (!input) return; + + if (open) { + if (event.key === "Escape") { + event.stopPropagation(); + event.preventDefault(); + return hide(); + } + if (event.key === "ArrowDown") { + event.preventDefault(); + return highlight(active + 1); + } + if (event.key === "ArrowUp") { + event.preventDefault(); + return highlight(active - 1); + } + if (event.key === "Enter" || event.key === "Tab") { + var all = options(); + if (all.length && active >= 0) { + /* Capture phase, so app.js's Enter-to-send never sees this one. + Without stopping it the message would be sent *and* the menu + item chosen. */ + event.preventDefault(); + event.stopPropagation(); + choose(all[active]); + return; + } + } + } + + if (event.key === "Enter" && !event.shiftKey && !event.isComposing) { + /* Not a menu key: a command typed in full and submitted. Handled here + rather than on submit so the form is never posted at all -- a + command is not a message and must not become one if it is unknown. */ + var command = commandIn(input.value); + if (command) { + event.preventDefault(); + event.stopPropagation(); + input.value = ""; + if (window.lembas && window.lembas.autosize) window.lembas.autosize(input); + runCommand(command.name, command.rest); + } + } + }, + true + ); + + document.addEventListener("input", function (event) { + if (event.target.closest("[data-composer-input]")) refresh(); + }); + + document.addEventListener("click", function (event) { + if (!event.target.closest(".composer-menu") && + !event.target.closest("[data-composer-input]")) hide(); + }); + + /* Opening the menu from a button, for anyone who would rather press than + type. Inserts the character and lets the normal path take over. */ + document.addEventListener("click", function (event) { + var button = event.target.closest("[data-mention-open]"); + if (!button) return; + event.preventDefault(); + var input = composer(); + if (!input) return; + input.focus(); + var caret = input.selectionStart; + var lead = caret && !/\s/.test(input.value[caret - 1]) ? " @" : "@"; + input.setRangeText(lead, caret, caret, "end"); + refresh(); + }); + + window.lembas = window.lembas || {}; + window.lembas.closeComposerMenu = hide; +})(); diff --git a/src/lembas/web/static/js/sw.js b/src/lembas/web/static/js/sw.js index 9caa673..1419998 100644 --- a/src/lembas/web/static/js/sw.js +++ b/src/lembas/web/static/js/sw.js @@ -28,6 +28,8 @@ var SHELL = [ "/static/css/admin.css", "/static/js/app.js", "/static/js/ui.js", + "/static/js/commands.js", + "/static/js/composer.js", "/static/js/audio.js", "/static/js/terminal.js", // Deliberately not the three xterm files below it: ~300KB precached on every diff --git a/src/lembas/web/templates/admin/agents.html b/src/lembas/web/templates/admin/agents.html index ab09015..499eabf 100644 --- a/src/lembas/web/templates/admin/agents.html +++ b/src/lembas/web/templates/admin/agents.html @@ -224,6 +224,42 @@ +
+

The project directory

+

+ A listing of the directory a chat works in, so a reply does not spend its + first rounds finding out what is there — and so files can be attached by + name with @. Built by one read-only command + (git ls-files where it works, otherwise find), + cached briefly, and shared by every chat pointed at the same directory. +

+ +
+ +

+ Off means no listing is built at all, and the file picker offers only + what is in the library. +

+
+ +
+ + +

+ This is spent on every request in an agent chat, so it is a + budget rather than a limit: directories that will not fit are shown as + a count and the model is told to look inside them itself. + 0 keeps the listing for the file picker and puts none + of it in the prompt. +

+
+
+
diff --git a/src/lembas/web/templates/base.html b/src/lembas/web/templates/base.html index c7a646f..508764f 100644 --- a/src/lembas/web/templates/base.html +++ b/src/lembas/web/templates/base.html @@ -54,6 +54,10 @@ +{# commands.js before composer.js: the second reads the first's table to draw + the `/` menu, and both are deferred so the order here is the run order. #} + + {# diff --git a/src/lembas/web/templates/chat/_composer.html b/src/lembas/web/templates/chat/_composer.html index ba1fc56..b4d3753 100644 --- a/src/lembas/web/templates/chat/_composer.html +++ b/src/lembas/web/templates/chat/_composer.html @@ -21,7 +21,13 @@ text they are in the same place as attach and send, which is where the hand already is. #} -
+{# The chat, the connection and the directory, where composer.js can read them + without parsing them back out of a URL. On a new chat the connection is + still being chosen, so the select and the hidden field win over these. #} +
{% if can.get("files.upload") %} {# Outside the form: it is only ever read by JavaScript, and inside it would be submitted as an empty file part on every message. @@ -117,6 +123,14 @@ {% endif %}
+ + {# The same menu the `@` key opens, for anyone who would rather press + than type. It inserts the character and gets out of the way. #} + {% endif %} @@ -243,6 +257,11 @@

Enter to send, Shift+Enter for a new line. + , + @ for files. {% if can.get("files.upload") %} Drag files in, or paste an image. {% if current_model and not current_model.capabilities_json.get("vision") %} diff --git a/src/lembas/web/templates/chat/_mention_picker.html b/src/lembas/web/templates/chat/_mention_picker.html new file mode 100644 index 0000000..7a9d7d4 --- /dev/null +++ b/src/lembas/web/templates/chat/_mention_picker.html @@ -0,0 +1,65 @@ +{% from "_macros.html" import icon %} +{# + What `@` offers. Two sources, one list, because somebody typing `@readme` is + not thinking about which store the answer is in. + + Swapped in whole on every keystroke, like the knowledge picker: the library + is searched with FTS rather than filtered in the browser, and the project + half is filtered beside it so the client stays one fetch and a list. + + Every name here came off somebody's filesystem or out of their library, so + all of it is escaped by autoescaping and none of it is marked safe. +#} +

+ {% if not files and not documents %} +

+ {% if q %} + Nothing matches “{{ q }}”. + {% else %} + Nothing to mention yet. + {% endif %} +

+ {% else %} + + {% if files %} +

In the project

+ + {% endif %} + + {% if documents %} +

In your library

+ + {% endif %} + + {% endif %} +
diff --git a/src/lembas/web/templates/chat/_message.html b/src/lembas/web/templates/chat/_message.html index 4912639..8cb5f5c 100644 --- a/src/lembas/web/templates/chat/_message.html +++ b/src/lembas/web/templates/chat/_message.html @@ -217,6 +217,7 @@ diff --git a/src/lembas/web/templates/chat/_usage.html b/src/lembas/web/templates/chat/_usage.html new file mode 100644 index 0000000..284c60f --- /dev/null +++ b/src/lembas/web/templates/chat/_usage.html @@ -0,0 +1,80 @@ +{# + What `/usage` shows. + + Two different numbers, deliberately kept apart. **Used** is how full the + window is right now -- one reply's prompt plus its completion, which is what + decides when compaction fires. **Spent** is everything this conversation has + cost end to end, which is larger and grows forever: a three-round reply pays + for its prompt three times but only ever occupies the window once. + + Anything derived from an estimate wears a `~`, because `services/tokens.py` + counts four characters to a token when an endpoint reports nothing, and a + precise-looking figure that is a guess is worse than an obvious guess. +#} + + + + + + + + {% if not metrics.context_limit %} + {# Unknown is not zero. Nobody has said how big this model's window is, so + the percentage is omitted rather than computed, and automatic + compaction never fires. #} + + + + + {% endif %} + + + + + + + + + + + + + + + {% if metrics.tokens_per_second %} + + + + + {% endif %} + + {% if chat.compact_summary %} + + + + + {% endif %} + +
In the window now + {% if metrics.context_tokens %} + {{ '~' if metrics.estimated }}{{ '{:,}'.format(metrics.context_tokens) }} tokens + {% if metrics.percent %} + — {{ metrics.percent }}% of + {{ '{:,}'.format(metrics.context_limit) }} + {% endif %} + {% else %} + Nothing yet. + {% endif %} +
Window size + Not recorded for {{ model.label if model else "this model" }}, so there is + no percentage and this chat will never compact itself. + {% if user.is_admin %} + Set it under Models. + {% endif %} +
Spent in total + {{ '~' if estimated }}{{ '{:,}'.format(totals.total) }} tokens + across {{ replies }} repl{{ 'y' if replies == 1 else 'ies' }} +
Of which sent{{ '~' if estimated }}{{ '{:,}'.format(totals.prompt) }} tokens
Of which written{{ '~' if estimated }}{{ '{:,}'.format(totals.completion) }} tokens
Last reply{{ '%.1f'|format(metrics.tokens_per_second) }} tokens/second
Compacted + Earlier turns are summarised. They are still in the transcript; they + just stop being sent. +
diff --git a/tests/conftest.py b/tests/conftest.py index 484f751..79b35a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -102,6 +102,21 @@ def fresh_terminal_registry() -> Iterator[None]: _clear() +@pytest.fixture(autouse=True) +def fresh_project_index() -> Iterator[None]: + """Empty the directory-listing cache between tests, for the third time. + + Keyed on (profile, directory) and both are recycled freely by fixtures, so + without this a test asserting "the listing said X" can be answered by the + previous test's walk of an entirely different tmp_path. + """ + from lembas.services.agent import index as index_service + + index_service.clear() + yield + index_service.clear() + + @pytest.fixture def db() -> Iterator[Session]: session = get_session_factory()() diff --git a/tests/test_agent_index.py b/tests/test_agent_index.py new file mode 100644 index 0000000..eb3f8d3 --- /dev/null +++ b/tests/test_agent_index.py @@ -0,0 +1,252 @@ +"""Listing a project directory: the ladder, the cache, and the budget. + +The ladder is exercised against a fake executor rather than a real host, +because what is being tested is *which rung is chosen* and what happens when +one falls through -- and a real box either has git or does not, which would +make the interesting cases unreachable. + +`scan_dir` itself is tested against a real SFTP server in test_agent_browse.py. +""" + +from __future__ import annotations + +import time + +import pytest + +from lembas.services.agent import index as index_service +from lembas.services.agent.base import ExecError, ExecResult, RemoteEntry + + +class _Fake: + """An executor that answers whatever the test says, and records the asking.""" + + def __init__(self, *, answers: dict[str, ExecResult] | None = None, tree=None): + self.answers = answers or {} + self.tree = tree or {} + self.commands: list[str] = [] + self.scans: list[str] = [] + + async def run(self, request) -> ExecResult: + self.commands.append(request.command) + for fragment, result in self.answers.items(): + if fragment in request.command: + return result + return ExecResult(exit_status=1, output="") + + async def scan_dir(self, path: str) -> list[RemoteEntry]: + self.scans.append(path) + if path not in self.tree: + raise ExecError(f"There is no directory at {path}.") + return self.tree[path] + + +def _ok(output: str) -> ExecResult: + return ExecResult(exit_status=0, output=output) + + +@pytest.fixture(autouse=True) +def _clean_cache(): + index_service.clear() + yield + index_service.clear() + + +# --- The ladder -------------------------------------------------------------- +async def test_git_is_tried_first_and_wins_where_it_works(): + """`--exclude-standard` is the whole reason. A repository already carries + somebody's considered list of what is not part of the project, and + reproducing it by hand is how an index ends up mostly build output.""" + executor = _Fake(answers={"git ls-files": _ok("README.md\nsrc/main.py\n")}) + + found = await index_service.build(executor, "/work") + + assert found.source == "git" + assert found.paths == ("README.md", "src/main.py") + assert executor.commands == [executor.commands[0]] + assert "--exclude-standard" in executor.commands[0] + + +async def test_find_takes_over_when_the_directory_is_not_a_repository(): + executor = _Fake(answers={"find .": _ok("./README.md\n./src/main.py\n")}) + + found = await index_service.build(executor, "/work") + + assert found.source == "find" + assert found.paths == ("README.md", "src/main.py") + assert len(executor.commands) == 2 # git was tried, and fell through + + +async def test_find_prunes_the_usual_noise(): + """Not applied to the git rung, which has already applied the repository's + own rules -- a checked-in `vendor/` is checked in on purpose.""" + executor = _Fake(answers={"find .": _ok("./a\n")}) + + await index_service.build(executor, "/work") + + assert "node_modules" in executor.commands[1] + assert "'.git'" in executor.commands[1] + + +async def test_sftp_is_the_last_resort_and_always_works(): + executor = _Fake( + tree={ + "/work": [RemoteEntry("src", True), RemoteEntry("README.md", False)], + "src": [RemoteEntry("main.py", False)], + } + ) + + found = await index_service.build(executor, "/work") + + assert found.source == "sftp" + assert "README.md" in found.paths + assert "src/main.py" in found.paths + + +async def test_a_directory_that_cannot_be_read_at_all_is_empty_not_an_error(): + """A reply must not fail because a listing did. The fragment carrying it + disappears instead, which is what `requires` is for.""" + executor = _Fake(tree={}) + + found = await index_service.build(executor, "/work") + + assert found.paths == () + assert not found.ok + + +async def test_control_characters_are_stripped_from_a_filename(): + """These end up inside a system prompt. An escape sequence in a filename + could otherwise repaint the transcript it is quoted in.""" + executor = _Fake(answers={"git ls-files": _ok("ok.py\nevil\x1b[31m.py\n")}) + + found = await index_service.build(executor, "/work") + + assert "evil[31m.py" in found.paths + assert "\x1b" not in "".join(found.paths) + + +async def test_the_number_of_paths_is_capped_and_says_so(): + """"There is nothing else here" and "I stopped looking" are different + answers, and it must not give the first for the second.""" + many = "\n".join(f"f{i}.py" for i in range(index_service.MAX_ENTRIES + 50)) + executor = _Fake(answers={"git ls-files": _ok(many)}) + + found = await index_service.build(executor, "/work") + + assert len(found.paths) == index_service.MAX_ENTRIES + assert found.truncated + + +# --- The cache --------------------------------------------------------------- +async def test_a_second_ask_does_not_walk_again(): + executor = _Fake(answers={"git ls-files": _ok("a.py\n")}) + + await index_service.ensure(executor, "profile-1", "/work") + await index_service.ensure(executor, "profile-1", "/work") + + assert len(executor.commands) == 1 + + +async def test_refreshing_walks_again(): + executor = _Fake(answers={"git ls-files": _ok("a.py\n")}) + + await index_service.ensure(executor, "profile-1", "/work") + await index_service.ensure(executor, "profile-1", "/work", refresh=True) + + assert len(executor.commands) == 2 + + +async def test_a_stale_index_is_not_returned(): + executor = _Fake(answers={"git ls-files": _ok("a.py\n")}) + await index_service.ensure(executor, "profile-1", "/work") + + index_service._CACHE[("profile-1", "/work")] = index_service.ProjectIndex( + paths=("a.py",), total=1, source="git", built_at=time.monotonic() - index_service.TTL - 1 + ) + + assert index_service.cached("profile-1", "/work") is None + + +async def test_forgetting_a_connection_drops_its_listings(): + """A connection somebody has just revoked must not leave a listing of the + machine behind it. Called from the same three places that close its + terminals.""" + executor = _Fake(answers={"git ls-files": _ok("a.py\n")}) + await index_service.ensure(executor, "profile-1", "/work") + await index_service.ensure(executor, "profile-2", "/work") + + assert index_service.forget("profile-1") == 1 + assert index_service.cached("profile-1", "/work") is None + assert index_service.cached("profile-2", "/work") is not None + + +def test_reading_the_cache_never_does_work(): + """`harness` calls this synchronously while assembling the system message, + so it must never be the thing that opens a connection.""" + assert index_service.cached("nobody", "/nowhere") is None + + +# --- The budget -------------------------------------------------------------- +def _index(paths) -> index_service.ProjectIndex: + return index_service.ProjectIndex( + paths=tuple(sorted(paths)), total=len(paths), source="git", built_at=time.monotonic() + ) + + +def test_a_small_tree_is_shown_whole(): + text = index_service.render(_index(["README.md", "src/main.py"]), 2000) + + assert "README.md" in text + assert "main.py" in text + assert "files)" not in text + + +def test_a_big_directory_becomes_a_count(): + paths = ["README.md"] + [f"vendor/f{i}.js" for i in range(400)] + + text = index_service.render(_index(paths), 300) + + assert "vendor/ (400 files)" in text + assert "README.md" in text + assert "file_list" in text + + +def test_the_deepest_big_directory_is_collapsed_first(): + """Collapsing by size alone takes `src/` before `src/.../vendor/` -- it is + bigger because it *contains* it -- and loses every name worth having in + order to fold away one directory of third-party files.""" + paths = [f"src/api/{n}.py" for n in "abcde"] + [ + f"src/web/vendor/f{i}.js" for i in range(300) + ] + + text = index_service.render(_index(paths), 400) + + assert "vendor/ (300 files)" in text + assert "a.py" in text # src/api survived + + +def test_a_flat_directory_of_thousands_is_still_bounded(): + """The one shape collapsing cannot help with: no directory to fold them + into, so the running budget has to bite instead.""" + text = index_service.render(_index([f"dump{i:05d}.log" for i in range(5000)]), 400) + + assert len(text) < 1200 + assert "more files" in text + + +def test_a_budget_of_zero_renders_nothing(): + """Which is how "index it for the picker, but say nothing to the model" is + expressed. The fragment vanishes rather than appearing empty.""" + assert index_service.render(_index(["a.py"]), 0) == "" + + +def test_an_empty_index_renders_nothing(): + assert index_service.render(index_service.ProjectIndex(), 2000) == "" + + +def test_a_truncated_index_says_it_is_a_sample(): + sample = index_service.ProjectIndex( + paths=("a.py", "b.py"), total=9000, truncated=True, source="find", built_at=time.monotonic() + ) + + assert "sample" in index_service.render(sample, 2000) diff --git a/tests/test_harness.py b/tests/test_harness.py index 7042121..c03875a 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -257,3 +257,103 @@ def test_the_tools_array_rides_along(db, owner): offered = tools_service.enabled_tools(db, chat, owner) body = chat_service.build_request(db, chat, tools=offered, user=owner) assert body["tools"] == offered + + +# --- The project listing ----------------------------------------------------- +# Injected from a cache that something else fills, because this module runs +# synchronously on the request path and an SFTP round trip here would hold a +# request open while somebody's machine thought about it. +def _agent_chat(db, owner): + from lembas.db.models import KIND_AGENT, SshProfile + + settings_store.update(db, {"enabled": True}, key=settings_store.AGENTS) + profile = SshProfile( + owner_id=owner.id, + name="Test box", + host="127.0.0.1", + port=22, + username="tester", + host_key="host key", + host_fingerprint="SHA256:x", + default_dir="/work", + ) + db.add(profile) + db.commit() + chat = Chat( + user_id=owner.id, + kind=KIND_AGENT, + ssh_profile_id=profile.id, + project_dir="/work", + ) + db.add(chat) + db.commit() + return chat, profile + + +def _agent_tools(db): + """The agent tools as offered, which REGISTRY does not carry. + + `REGISTRY` is built at import time and holds the built-ins alone; the agent + tools are listed by `registry(db)`, unbound to any chat. That is the same + lookup the harness does to map `shell_run` back to the `agent` family, and + the reason it exists at all. + """ + book = tools_service.registry(db) + return [book["shell_run"].schema] + + +def _cache(profile_id, paths): + import time + + from lembas.services.agent import index as index_service + + index_service._CACHE[(profile_id, "/work")] = index_service.ProjectIndex( + paths=tuple(paths), total=len(paths), source="git", built_at=time.monotonic() + ) + + +def test_the_project_listing_reaches_the_model(db, owner): + chat, profile = _agent_chat(db, owner) + _cache(profile.id, ["README.md", "src/main.py"]) + + text = harness.compose(db, owner, _agent_tools(db), chat=chat) + + assert "Files in /work" in text + assert "README.md" in text + + +def test_nothing_cached_means_no_section_at_all(db, owner): + """Not an empty heading. `Fragment.requires` makes the whole thing vanish, + which is what lets the first reply in a new chat outrun the first walk + without saying anything strange.""" + chat, _profile = _agent_chat(db, owner) + + text = harness.compose(db, owner, _agent_tools(db), chat=chat) + + assert "Files in" not in text + + +def test_a_budget_of_zero_keeps_the_listing_out_of_the_prompt(db, owner): + """The listing is still built and the file picker still uses it. This is + the only way to say "index it, but do not spend context on it".""" + chat, profile = _agent_chat(db, owner) + _cache(profile.id, ["README.md"]) + settings_store.update(db, {"index_chars": 0}, key=settings_store.AGENTS) + + assert "Files in" not in harness.compose(db, owner, _agent_tools(db), chat=chat) + + +def test_switching_the_listing_off_keeps_it_out(db, owner): + chat, profile = _agent_chat(db, owner) + _cache(profile.id, ["README.md"]) + settings_store.update(db, {"index_enabled": False}, key=settings_store.AGENTS) + + assert "Files in" not in harness.compose(db, owner, _agent_tools(db), chat=chat) + + +def test_a_plain_chat_is_told_nothing_about_files(db, owner): + chat = Chat(user_id=owner.id) + db.add(chat) + db.commit() + + assert "Files in" not in harness.compose(db, owner, _tools("web_search"), chat=chat) diff --git a/tests/test_mentions.py b/tests/test_mentions.py new file mode 100644 index 0000000..bbe8af2 --- /dev/null +++ b/tests/test_mentions.py @@ -0,0 +1,331 @@ +"""`@` attachments: what the picker offers, and what the model is told it got. + +The point of the second half is the one the feature was asked for: a model +handed a file called `main.py` cannot tell which of four it is looking at, and +cannot name it back when asked to change something. So the path and the machine +travel with the contents. +""" + +from __future__ import annotations + +import time + +import pytest +from fastapi.testclient import TestClient +from sqlalchemy import select + +from lembas.db.models import Attachment, Message, SshProfile, User +from lembas.services import chat as chat_service +from lembas.services import settings_store +from lembas.services.agent import index as index_service +from lembas.services.agent import ssh as ssh_service + +asyncssh = pytest.importorskip("asyncssh") + + +class _Server(asyncssh.SSHServer): + def begin_auth(self, username: str) -> bool: + return False + + +@pytest.fixture +def box(tmp_path): + """A real SFTP server on its own loop, with a small tree to mention from.""" + import asyncio + import threading + + root = tmp_path / "work" + (root / "src").mkdir(parents=True) + (root / "src" / "main.py").write_text("print('hello')\n") + (root / "README.md").write_text("# Project\n") + + loop = asyncio.new_event_loop() + thread = threading.Thread(target=loop.run_forever, daemon=True) + thread.start() + + async def start(): + server = await asyncssh.create_server( + _Server, + "127.0.0.1", + 0, + server_host_keys=[asyncssh.generate_private_key("ssh-ed25519")], + sftp_factory=True, + ) + port = next(iter(server.sockets)).getsockname()[1] + line, fingerprint = await ssh_service.capture_host_key("127.0.0.1", port) + return server, port, line, fingerprint + + server, port, line, fingerprint = asyncio.run_coroutine_threadsafe(start(), loop).result(10) + try: + yield {"port": port, "host_key": line, "fingerprint": fingerprint, "root": str(root)} + finally: + + async def stop(): + server.close() + await server.wait_closed() + + asyncio.run_coroutine_threadsafe(stop(), loop).result(10) + loop.call_soon_threadsafe(loop.stop) + thread.join(timeout=5) + + +def _profile(db, box) -> SshProfile: + settings_store.update(db, {"enabled": True}, key=settings_store.AGENTS) + user = db.scalars(select(User)).first() + profile = SshProfile( + owner_id=user.id, + name="Container", + host="127.0.0.1", + port=box["port"], + username="tester", + host_key=box["host_key"], + host_fingerprint=box["fingerprint"], + default_dir=box["root"], + ) + db.add(profile) + db.commit() + return profile + + +def _index(profile, box, paths=("README.md", "src/main.py")): + index_service._CACHE[(profile.id, box["root"])] = index_service.ProjectIndex( + paths=tuple(paths), total=len(paths), source="git", built_at=time.monotonic() + ) + + +# --- The picker -------------------------------------------------------------- +def test_the_picker_offers_project_files(client: TestClient, db, registered, box): + profile = _profile(db, box) + _index(profile, box) + + body = client.get( + "/api/files/mention-picker", + params={"q": "main", "profile_id": profile.id, "project_dir": box["root"]}, + ).text + + assert "src/main.py" in body + assert "README.md" not in body # filtered by the query + + +def test_the_picker_never_waits_on_a_machine(client: TestClient, db, registered, box): + """No listing built yet means no files offered, not a connection opened. + + This is a keystroke-latency path. Building the index here would put an SSH + round trip between a letter and the menu. + """ + profile = _profile(db, box) + + body = client.get( + "/api/files/mention-picker", + params={"profile_id": profile.id, "project_dir": box["root"]}, + ).text + + assert "main.py" not in body + + +def test_the_picker_refuses_somebody_elses_connection(client: TestClient, db, registered, box): + """An id in a query string is not an authorisation, and this lists the + contents of somebody's machine.""" + profile = _profile(db, box) + _index(profile, box) + + client.post("/auth/logout", follow_redirects=False) + client.post( + "/auth/register", + data={"name": "Sam", "email": "sam@shire.test", "password": "potatoes-po-ta-toes"}, + follow_redirects=False, + ) + + body = client.get( + "/api/files/mention-picker", + params={"profile_id": profile.id, "project_dir": box["root"]}, + ).text + + assert "main.py" not in body + + +def test_a_plain_chat_gets_a_picker_with_no_file_half(client: TestClient, db, registered): + """`@` works everywhere; only the project half needs a connection.""" + response = client.get("/api/files/mention-picker") + + assert response.status_code == 200 + assert "In the project" not in response.text + + +# --- Attaching --------------------------------------------------------------- +def test_a_mentioned_file_arrives_with_its_contents(client: TestClient, db, registered, box): + profile = _profile(db, box) + + client.post( + "/api/files/from-project", + data={"profile_id": profile.id, "path": "src/main.py"}, + ) + + attachment = db.scalars(select(Attachment)).one() + assert "print('hello')" in attachment.extracted_text + assert attachment.filename == "main.py" + + +def test_the_model_is_told_which_file_and_where(client: TestClient, db, registered, box): + """The whole reason the columns exist. `main.py` alone is not an answer to + "which one", and a model cannot name a file back that it was never given + the path of.""" + profile = _profile(db, box) + client.post( + "/api/files/from-project", + data={"profile_id": profile.id, "path": "src/main.py"}, + ) + + attachment = db.scalars(select(Attachment)).one() + message = Message(chat_id=None, role="user", content="") + message.attachments = [attachment] + + block = chat_service.document_context(message) + + assert 'path="src/main.py"' in block + assert 'from="Container"' in block + assert 'name="main.py"' in block + + +def test_a_quote_in_a_path_cannot_break_out_of_the_tag(client: TestClient, db, registered): + """These are attribute values in a tag we write. A path containing a quote + would otherwise close it early and the rest would read as instructions.""" + from lembas.services import files as files_service + + user = db.scalars(select(User)).first() + attachment = files_service.store_text( + db, + user_id=user.id, + chat_id=None, + filename="x.txt", + text="body", + source_path='/tmp/a">