diff --git a/README.md b/README.md index 06641af..95b4d92 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,9 @@ runtime. Clone it, `pip install -e .`, run it. - **Reasoning display** — thinking from reasoning models streams into its own collapsible block, labelled with how long it took, and is never replayed as context +- **Attachments** — drag, paste or pick images, PDFs and text files. Images are + downscaled and sent to vision models; PDF and text content is extracted and + put in the prompt - **Folders** — arbitrarily nested, delete a folder without losing the chats inside it - **OpenAI connections** — point at OpenAI, LM Studio, vLLM, llama.cpp, @@ -53,8 +56,8 @@ runtime. Clone it, `pip install -e .`, run it. **Planned** -File upload, vision and PDFs · built-in tools with admin settings · custom tools -and MCP servers · agentic execution (local and over SSH) · image generation. +Built-in tools with admin settings · custom tools and MCP servers · agentic +execution (local and over SSH) · image generation · OCR for scanned PDFs. ## Quick start diff --git a/pyproject.toml b/pyproject.toml index ba6dd0b..0b28960 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,8 @@ dependencies = [ "linkify-it-py>=2.0", # bare URLs in model output become links "pygments>=2.18", "nh3>=0.2.18", + "pypdf>=5.1", # PDF text extraction for attachments + "pillow>=11.0", # image validation and downscaling for vision "typer>=0.12", ] diff --git a/src/lembas/api/chats.py b/src/lembas/api/chats.py index c5a941c..a825183 100644 --- a/src/lembas/api/chats.py +++ b/src/lembas/api/chats.py @@ -16,6 +16,7 @@ from lembas.db.models import ROLE_ASSISTANT, ROLE_USER, Chat, Message, User from lembas.db.session import session_scope from lembas.security import permissions from lembas.services import chat as chat_service +from lembas.services import files as files_service from lembas.services import sse from lembas.services.llm.openai_client import ( LLMError, @@ -66,7 +67,8 @@ async def post_message( db: Db, user: RequiredUser, chat_id: str, - content: str = Form(...), + content: str = Form(""), + file_ids: list[str] = Form(default=[]), ) -> Response: """Persist the user's turn and hand back the pair of bubbles. @@ -77,10 +79,15 @@ async def post_message( chat = _owned_chat(db, chat_id, user.id) content = content.strip() - if not content: + # "Here, look at this" with no words is a legitimate turn, so an empty + # message is only empty when it carries nothing at all. + if not content and not file_ids: return Response(status_code=status.HTTP_204_NO_CONTENT) user_message = chat_service.create_message(db, chat, ROLE_USER, content) + if file_ids: + files_service.claim(db, ids=file_ids, user_id=user.id, message_id=user_message.id) + db.refresh(user_message) assistant_message = chat_service.create_message( db, chat, ROLE_ASSISTANT, "", complete_=False, model_id=chat.model_id ) @@ -130,6 +137,19 @@ async def stream_message( ) +def _plain_text(content: str | list) -> str: + """The text of a message payload, whether it is a string or content parts.""" + if isinstance(content, str): + return content + if isinstance(content, list): + return " ".join( + part.get("text", "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ).strip() + return "" + + async def _generate(chat_id: str, message_id: str) -> AsyncIterator[str]: """Drive one completion and frame it as SSE. @@ -159,8 +179,14 @@ async def _generate(chat_id: str, message_id: str) -> AsyncIterator[str]: try: endpoint, model_id = chat_service.resolve_endpoint(db, chat) payload = chat_service.build_request(db, chat, upto=message) + # A multimodal turn's content is a list of parts, not a string, so + # the text has to be picked out before it can title a chat. first_user_text = next( - (m["content"] for m in reversed(payload["messages"]) if m["role"] == ROLE_USER), + ( + _plain_text(m["content"]) + for m in reversed(payload["messages"]) + if m["role"] == ROLE_USER + ), "", ) diff --git a/src/lembas/api/files.py b/src/lembas/api/files.py new file mode 100644 index 0000000..5d6e22f --- /dev/null +++ b/src/lembas/api/files.py @@ -0,0 +1,117 @@ +"""Uploading, serving and removing chat attachments.""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status +from fastapi.responses import FileResponse + +from lembas.api.deps import Db, RequiredUser, require_permission +from lembas.db.models import Attachment +from lembas.services import files as files_service +from lembas.web.templating import templates + +log = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/files", tags=["files"]) + + +def _owned(db: Db, attachment_id: str, user_id: str) -> Attachment: + attachment = db.get(Attachment, attachment_id) + if attachment is None or attachment.user_id != user_id: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That file no longer exists.") + return attachment + + +@router.post("", dependencies=[Depends(require_permission("files.upload"))]) +async def upload( + request: Request, + db: Db, + user: RequiredUser, + file: UploadFile = File(...), + chat_id: str = "", +) -> Response: + """Accept one file and return the chip that represents it in the composer. + + The attachment is stored immediately but left unbound: it only joins a + message when that message is sent. That is what lets a file be removed + before sending, and what the orphan sweep later cleans up. + """ + payload = await file.read() + + try: + attachment = files_service.store( + db, + user_id=user.id, + chat_id=chat_id or None, + payload=payload, + filename=file.filename or "file", + ) + except files_service.FileError as exc: + # 200 with an error chip rather than a 4xx: htmx swaps the response + # body either way, and an error the user can read beats a silent + # failure in the console. + return templates.TemplateResponse( + request, + "chat/_attachment_error.html", + {"request": request, "filename": file.filename or "file", "error": str(exc)}, + ) + + return templates.TemplateResponse( + request, + "chat/_attachment_chip.html", + {"request": request, "attachment": attachment}, + ) + + +@router.delete("/{attachment_id}") +async def remove(db: Db, user: RequiredUser, attachment_id: str) -> Response: + """Detach a file before it has been sent.""" + attachment = _owned(db, attachment_id, user.id) + if attachment.message_id is not None: + # Deleting it now would rewrite a conversation that has already been + # sent to a model and read by the user. + raise HTTPException( + status.HTTP_409_CONFLICT, "That file is part of a sent message." + ) + files_service.delete(db, attachment) + return Response(status_code=status.HTTP_200_OK) + + +@router.get("/{attachment_id}/content") +async def content(db: Db, user: RequiredUser, attachment_id: str) -> Response: + """Serve an attachment back to its owner.""" + attachment = _owned(db, attachment_id, user.id) + path = files_service.stored_path(attachment.stored_name) + if path is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That file is no longer on disk.") + + # inline for images so they render in the thread; attachment for everything + # else so a text/html upload can never be executed in this origin. + disposition = "inline" if attachment.is_image else "attachment" + return FileResponse( + path, + media_type=attachment.media_type if attachment.is_image else "application/octet-stream", + headers={ + "Content-Disposition": f'{disposition}; filename="{attachment.filename}"', + "Cache-Control": "private, max-age=604800", + # Belt and braces: even for images, never let a browser sniff its + # way to treating the bytes as something executable. + "X-Content-Type-Options": "nosniff", + }, + ) + + +@router.get("/{attachment_id}/text") +async def extracted_text(db: Db, user: RequiredUser, attachment_id: str) -> Response: + """The text a document contributed to the prompt. + + Worth being able to see: a PDF that extracted badly explains a strange + reply, and there is otherwise no way to tell what the model was given. + """ + attachment = _owned(db, attachment_id, user.id) + return Response( + attachment.extracted_text or attachment.extraction_error, + media_type="text/plain; charset=utf-8", + ) diff --git a/src/lembas/db/models/__init__.py b/src/lembas/db/models/__init__.py index d6f51ec..4a8e506 100644 --- a/src/lembas/db/models/__init__.py +++ b/src/lembas/db/models/__init__.py @@ -5,6 +5,12 @@ what ``init_db()`` relies on to create the schema at startup. Any new model module must be imported here or its table will silently never be created. """ +from lembas.db.models.attachment import ( + KIND_DOCUMENT, + KIND_IMAGE, + KIND_TEXT, + Attachment, +) from lembas.db.models.chat import ( ROLE_ASSISTANT, ROLE_SYSTEM, @@ -26,6 +32,10 @@ from lembas.db.models.user import ( ) __all__ = [ + "Attachment", + "KIND_DOCUMENT", + "KIND_IMAGE", + "KIND_TEXT", "ROLE_ADMIN", "ROLE_ASSISTANT", "ROLE_PENDING", diff --git a/src/lembas/db/models/attachment.py b/src/lembas/db/models/attachment.py new file mode 100644 index 0000000..af7bc99 --- /dev/null +++ b/src/lembas/db/models/attachment.py @@ -0,0 +1,74 @@ +"""Files attached to chat messages.""" + +from __future__ import annotations + +from sqlalchemy import Boolean, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from lembas.db.base import Base, Timestamps, UUIDPrimaryKey + +# What the file is for, decided at upload time. Drives both how it is rendered +# and how it reaches the model: images become multimodal parts, everything else +# becomes text in the prompt. +KIND_IMAGE = "image" +KIND_DOCUMENT = "document" # PDF: text is extracted +KIND_TEXT = "text" # plain text, markdown, csv, source code + + +class Attachment(UUIDPrimaryKey, Timestamps, Base): + __tablename__ = "attachments" + + user_id: Mapped[str] = mapped_column( + String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + chat_id: Mapped[str | None] = mapped_column( + String(32), ForeignKey("chats.id", ondelete="CASCADE"), index=True + ) + # Null while the file is uploaded but the message has not been sent yet. + # Those orphans are swept periodically -- see services.files.sweep_orphans. + message_id: Mapped[str | None] = mapped_column( + String(32), ForeignKey("messages.id", ondelete="CASCADE"), index=True + ) + + # What the uploader called it. Display only, never used as a path. + filename: Mapped[str] = mapped_column(String(300), nullable=False) + # Random name on disk. See services.files for why the two are separate. + stored_name: Mapped[str] = mapped_column(String(120), nullable=False) + + media_type: Mapped[str] = mapped_column(String(100), default="") + size_bytes: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + kind: Mapped[str] = mapped_column(String(16), default=KIND_DOCUMENT, nullable=False) + + # Images only, after downscaling. + width: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + height: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + + # Documents and text: the content that actually reaches the model. Held in + # the database rather than re-extracted per request -- extraction is slow, + # and a reply must not silently change because a PDF parser was upgraded. + extracted_text: Mapped[str] = mapped_column(Text, default="") + pages: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + truncated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + + # Non-empty when the file was stored but its text could not be read, e.g. a + # scanned PDF with no text layer. Shown next to the attachment so the user + # is not left wondering why the model ignored it. + extraction_error: Mapped[str] = mapped_column(Text, default="") + + message: Mapped[Message] = relationship(back_populates="attachments") # noqa: F821 + + @property + def is_image(self) -> bool: + return self.kind == KIND_IMAGE + + @property + def human_size(self) -> str: + size = float(self.size_bytes) + for unit in ("B", "KB", "MB"): + if size < 1024 or unit == "MB": + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} MB" + + def __repr__(self) -> str: + return f"" diff --git a/src/lembas/db/models/chat.py b/src/lembas/db/models/chat.py index 9f4f5c2..a44c158 100644 --- a/src/lembas/db/models/chat.py +++ b/src/lembas/db/models/chat.py @@ -120,6 +120,19 @@ class Message(UUIDPrimaryKey, Timestamps, Base): complete: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) chat: Mapped[Chat] = relationship(back_populates="messages") + attachments: Mapped[list[Attachment]] = relationship( # noqa: F821 + back_populates="message", + cascade="all, delete-orphan", + order_by="Attachment.created_at", + ) + + @property + def images(self) -> list: + return [a for a in self.attachments if a.is_image] + + @property + def documents(self) -> list: + return [a for a in self.attachments if not a.is_image] def __repr__(self) -> str: return f"" diff --git a/src/lembas/main.py b/src/lembas/main.py index b3e694d..cf3d7d5 100644 --- a/src/lembas/main.py +++ b/src/lembas/main.py @@ -18,6 +18,7 @@ from lembas.api import ( admin_users, auth, chats, + files, folders, pages, preferences, @@ -52,6 +53,17 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: ' python -c "import secrets; print(secrets.token_urlsafe(48))"' ) + # Files chosen in a composer that was never sent would otherwise sit on + # disk forever. Cheap, and startup is the natural moment for it. + try: + from lembas.db.session import session_scope + from lembas.services.files import sweep_orphans + + with session_scope() as db: + sweep_orphans(db) + except Exception: # noqa: BLE001 - housekeeping must never block startup + log.exception("orphaned upload sweep failed") + log.info("LLeMbas %s starting on http://%s:%s", __version__, settings.host, settings.port) log.info("data directory: %s", settings.data_dir.resolve()) yield @@ -74,6 +86,7 @@ def create_app() -> FastAPI: app.include_router(auth.router) app.include_router(preferences.router) app.include_router(chats.router) + app.include_router(files.router) app.include_router(folders.router) app.include_router(admin.router) app.include_router(admin_users.router) diff --git a/src/lembas/security/permissions.py b/src/lembas/security/permissions.py index f3385dd..8ec2544 100644 --- a/src/lembas/security/permissions.py +++ b/src/lembas/security/permissions.py @@ -70,6 +70,14 @@ PERMISSION_DEFS: tuple[PermissionDef, ...] = ( True, "Workspace", ), + PermissionDef( + "files.upload", + "Attach files", + "Attach images, PDFs and text files to a message. Images only reach " + "models marked as having vision.", + True, + "Workspace", + ), ) PERMISSION_KEYS = tuple(d.key for d in PERMISSION_DEFS) diff --git a/src/lembas/services/chat.py b/src/lembas/services/chat.py index e7151de..5eb66f0 100644 --- a/src/lembas/services/chat.py +++ b/src/lembas/services/chat.py @@ -17,6 +17,7 @@ from lembas.db.models import ( Message, Model, ) +from lembas.services import files as files_service from lembas.services.llm.openai_client import Endpoint, LLMError, complete log = logging.getLogger(__name__) @@ -71,7 +72,66 @@ def resolve_endpoint(db: DBSession, chat: Chat) -> tuple[Endpoint, str]: return Endpoint.from_connection(connection), chat.model_id -def build_messages(db: DBSession, chat: Chat, *, upto: Message | None = None) -> list[dict]: +def document_context(message: Message) -> str: + """Extracted text from a message's non-image attachments. + + Wrapped in named tags so the model can tell one document from another, and + tell all of them from what the user actually typed. Truncation is stated + inline rather than silently, so a model asked about page 400 of a 300-page + extract can say it did not see it. + """ + blocks: list[str] = [] + for attachment in message.documents: + if not attachment.extracted_text.strip(): + continue + note = " (truncated)" if attachment.truncated else "" + blocks.append( + f'\n' + f"{attachment.extracted_text.strip()}\n" + f"" + ) + return "\n\n".join(blocks) + + +def message_payload(message: Message, *, vision: bool) -> dict[str, Any]: + """One history entry in the shape the endpoint expects. + + Plain text stays a plain string: sending the multimodal list form to an + endpoint that does not implement it is a reliable way to get a 400, and + most local runners do not. + """ + text = message.content.strip() + + documents = document_context(message) + if documents: + # Documents lead so the question that follows has its material already + # in view, which is how these models are trained to read a prompt. + text = f"{documents}\n\n{text}" if text else documents + + images = message.images if vision else [] + if not images: + return {"role": message.role, "content": text} + + parts: list[dict[str, Any]] = [] + if text: + parts.append({"type": "text", "text": text}) + for attachment in images: + uri = files_service.data_uri(attachment) + if uri is None: + # The row survived but the file did not. Better to say so than to + # send a turn that silently lost its picture. + log.warning("attachment %s has no file on disk", attachment.id) + continue + parts.append({"type": "image_url", "image_url": {"url": uri}}) + + if not parts: + return {"role": message.role, "content": text} + return {"role": message.role, "content": parts} + + +def build_messages( + db: DBSession, chat: Chat, *, upto: Message | None = None, vision: bool = False +) -> list[dict]: """Assemble the message list to send upstream. `upto` excludes the placeholder assistant row being generated into, and @@ -88,24 +148,38 @@ def build_messages(db: DBSession, chat: Chat, *, upto: Message | None = None) -> for message in history: if upto is not None and message.id == upto.id: break - # Skip turns that failed or produced nothing: sending an empty - # assistant message upsets several providers. - if message.error or not message.content.strip(): + # Skip turns that failed or produced nothing -- but a message carrying + # only an attachment has no text and must still be sent. + if message.error: continue - payload.append({"role": message.role, "content": message.content}) + if not message.content.strip() and not message.attachments: + continue + payload.append(message_payload(message, vision=vision)) return payload +def model_supports(db: DBSession, chat: Chat, capability: str) -> bool: + """Whether the chat's current model is marked as having a capability.""" + model = db.scalar( + select(Model).where(Model.model_id == chat.model_id).order_by(Model.position) + ) + return bool(model and (model.capabilities_json or {}).get(capability)) + + def build_request(db: DBSession, chat: Chat, *, upto: Message | None = None) -> dict[str, Any]: params = { key: value for key, value in (chat.params_json or {}).items() if key in FORWARDED_PARAMS and value not in (None, "") } + # Images are only sent to a model an administrator has marked as having + # vision. Sending them to one that has not is not a graceful degradation: + # most endpoints reject the whole request. + vision = model_supports(db, chat, "vision") return { "model": chat.model_id, - "messages": build_messages(db, chat, upto=upto), + "messages": build_messages(db, chat, upto=upto, vision=vision), **params, } diff --git a/src/lembas/services/files.py b/src/lembas/services/files.py new file mode 100644 index 0000000..06eb6d8 --- /dev/null +++ b/src/lembas/services/files.py @@ -0,0 +1,400 @@ +"""Storing and reading uploaded attachments. + +Three kinds of file, each handled differently on the way to the model: + +* **Images** are downscaled and re-encoded, then sent as multimodal content + parts. Downscaling is not cosmetic -- a phone photo is several megabytes of + base64, which is both slow and a large slice of the context window. +* **PDFs** have their text extracted once, at upload. Extraction is slow and a + reply must not silently change because a parser was upgraded later. +* **Plain text** (including source code and CSV) is decoded and stored as-is. + +Everything an uploader supplies is treated as hostile: the type is decided by +inspecting the bytes rather than trusting the browser, the name on disk is +random, and both image dimensions and PDF page counts are capped so a small +file cannot expand into an enormous amount of work. +""" + +from __future__ import annotations + +import io +import logging +import secrets +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from PIL import Image, UnidentifiedImageError +from sqlalchemy import select +from sqlalchemy.orm import Session as DBSession + +from lembas.config import settings +from lembas.db.models import KIND_DOCUMENT, KIND_IMAGE, KIND_TEXT, Attachment + +log = logging.getLogger(__name__) + +# --- Limits ------------------------------------------------------------------ +MAX_UPLOAD_BYTES = 20 * 1024 * 1024 + +# Longest edge after downscaling. Large enough for a model to read a screenshot +# or a page of text, small enough that the base64 stays reasonable. +MAX_IMAGE_EDGE = 1400 +JPEG_QUALITY = 85 + +# Pillow's own guard against decompression bombs: a 60,000x60,000 PNG is a few +# KB on disk and hundreds of GB decoded. +Image.MAX_IMAGE_PIXELS = 64_000_000 + +MAX_PDF_PAGES = 300 +# Characters of extracted text kept per document. Roughly 30k tokens, which is +# already a large slice of most context windows; more is rarely useful and +# frequently breaks the request outright. +MAX_EXTRACTED_CHARS = 120_000 + +# Orphans are files uploaded into a composer that was never sent. +ORPHAN_AGE = timedelta(hours=24) + +IMAGE_TYPES: dict[bytes, tuple[str, str]] = { + b"\x89PNG\r\n\x1a\n": ("image/png", ".png"), + b"\xff\xd8\xff": ("image/jpeg", ".jpg"), + b"GIF87a": ("image/gif", ".gif"), + b"GIF89a": ("image/gif", ".gif"), +} + +# Extensions treated as text when the bytes decode cleanly as UTF-8. The list +# exists only to pick a sensible media type; decodability is what actually +# decides, so an unlisted extension still works. +TEXT_EXTENSIONS = { + ".txt": "text/plain", ".md": "text/markdown", ".markdown": "text/markdown", + ".csv": "text/csv", ".tsv": "text/tab-separated-values", + ".json": "application/json", ".yaml": "text/yaml", ".yml": "text/yaml", + ".toml": "text/toml", ".ini": "text/plain", ".cfg": "text/plain", + ".xml": "text/xml", ".html": "text/plain", ".css": "text/plain", + ".py": "text/x-python", ".js": "text/javascript", ".ts": "text/typescript", + ".rs": "text/x-rust", ".go": "text/x-go", ".c": "text/x-c", ".h": "text/x-c", + ".cpp": "text/x-c++", ".java": "text/x-java", ".rb": "text/x-ruby", + ".sh": "text/x-shellscript", ".sql": "text/x-sql", ".log": "text/plain", +} + + +class FileError(Exception): + """A rejected upload, with a message fit to show the user.""" + + +@dataclass +class Prepared: + """The result of inspecting and processing an upload, before it is stored.""" + + payload: bytes + kind: str + media_type: str + extension: str + width: int = 0 + height: int = 0 + extracted_text: str = "" + pages: int = 0 + truncated: bool = False + extraction_error: str = "" + + +# --- Storage ----------------------------------------------------------------- +def attachments_dir() -> Path: + path = settings.uploads_dir / "attachments" + path.mkdir(parents=True, exist_ok=True) + return path + + +def stored_path(stored_name: str) -> Path | None: + """Resolve a stored name to a path, refusing anything outside the directory.""" + if not stored_name or "/" in stored_name or "\\" in stored_name or stored_name.startswith("."): + return None + base = attachments_dir().resolve() + path = (base / stored_name).resolve() + try: + path.relative_to(base) + except ValueError: + return None + return path if path.is_file() else None + + +# --- Type detection ---------------------------------------------------------- +def _detect_image(payload: bytes) -> tuple[str, str] | None: + for signature, (media_type, extension) in IMAGE_TYPES.items(): + if payload.startswith(signature): + return media_type, extension + if payload[:4] == b"RIFF" and payload[8:12] == b"WEBP": + return "image/webp", ".webp" + return None + + +def _looks_like_pdf(payload: bytes) -> bool: + # The header is allowed a little leading junk by the spec, and real files + # in the wild use it. + return b"%PDF-" in payload[:1024] + + +# --- Processing -------------------------------------------------------------- +def _process_image(payload: bytes) -> Prepared: + try: + with Image.open(io.BytesIO(payload)) as image: + image.load() + has_alpha = image.mode in ("RGBA", "LA", "P") and "transparency" in image.info + # Animation is lost on re-encode; keeping only the first frame is + # honest and is what a model would see anyway. + frame = image.convert("RGBA" if has_alpha else "RGB") + + width, height = frame.size + longest = max(width, height) + if longest > MAX_IMAGE_EDGE: + scale = MAX_IMAGE_EDGE / longest + frame = frame.resize( + (max(1, int(width * scale)), max(1, int(height * scale))), + Image.LANCZOS, + ) + + buffer = io.BytesIO() + if has_alpha: + frame.save(buffer, format="PNG", optimize=True) + media_type, extension = "image/png", ".png" + else: + frame.save(buffer, format="JPEG", quality=JPEG_QUALITY, optimize=True) + media_type, extension = "image/jpeg", ".jpg" + + return Prepared( + payload=buffer.getvalue(), + kind=KIND_IMAGE, + media_type=media_type, + extension=extension, + width=frame.width, + height=frame.height, + ) + except Image.DecompressionBombError as exc: + raise FileError("That image's dimensions are implausibly large.") from exc + except (UnidentifiedImageError, OSError, ValueError) as exc: + raise FileError("That image could not be read. Is it corrupt?") from exc + + +def _process_pdf(payload: bytes) -> Prepared: + from pypdf import PdfReader + from pypdf.errors import PdfReadError + + prepared = Prepared( + payload=payload, kind=KIND_DOCUMENT, media_type="application/pdf", extension=".pdf" + ) + + try: + reader = PdfReader(io.BytesIO(payload)) + if reader.is_encrypted: + # An empty password unlocks a surprising number of "encrypted" PDFs. + try: + reader.decrypt("") + except Exception: # noqa: BLE001 - any failure means the same thing + prepared.extraction_error = ( + "This PDF is password-protected, so its text could not be read." + ) + return prepared + + prepared.pages = len(reader.pages) + chunks: list[str] = [] + total = 0 + + for index, page in enumerate(reader.pages[:MAX_PDF_PAGES]): + try: + text = page.extract_text() or "" + except Exception as exc: # noqa: BLE001 - one bad page is not fatal + log.debug("page %d of a PDF failed to extract: %s", index, exc) + continue + if not text.strip(): + continue + chunks.append(f"[page {index + 1}]\n{text.strip()}") + total += len(text) + if total >= MAX_EXTRACTED_CHARS: + prepared.truncated = True + break + + if prepared.pages > MAX_PDF_PAGES: + prepared.truncated = True + + prepared.extracted_text = "\n\n".join(chunks)[:MAX_EXTRACTED_CHARS] + + if not prepared.extracted_text.strip(): + # Almost always a scan. Saying so beats the model silently ignoring + # a document the user believes it can read. + prepared.extraction_error = ( + "No text could be extracted. This looks like a scanned PDF; " + "LLeMbas does not do OCR yet." + ) + + except PdfReadError as exc: + prepared.extraction_error = "This file is not a readable PDF." + log.info("unreadable PDF: %s", exc) + except Exception as exc: # noqa: BLE001 - never let a bad file 500 the upload + prepared.extraction_error = "This PDF could not be read." + log.warning("unexpected PDF failure: %s", exc) + + return prepared + + +def _process_text(payload: bytes, filename: str) -> Prepared: + for encoding in ("utf-8", "utf-16", "latin-1"): + try: + text = payload.decode(encoding) + break + except (UnicodeDecodeError, LookupError): + continue + else: + raise FileError("That file is not text, and is not a format LLeMbas can read.") + + # Null bytes mean this decoded by luck (latin-1 decodes any byte) and is + # really a binary file. + if "\x00" in text[:4096]: + raise FileError("That file is not text, and is not a format LLeMbas can read.") + + truncated = len(text) > MAX_EXTRACTED_CHARS + extension = Path(filename).suffix.lower() + + return Prepared( + payload=payload, + kind=KIND_TEXT, + media_type=TEXT_EXTENSIONS.get(extension, "text/plain"), + extension=extension if extension in TEXT_EXTENSIONS else ".txt", + extracted_text=text[:MAX_EXTRACTED_CHARS], + truncated=truncated, + ) + + +def prepare(payload: bytes, filename: str) -> Prepared: + """Inspect an upload, decide what it is, and process it accordingly.""" + if not payload: + raise FileError("That file is empty.") + if len(payload) > MAX_UPLOAD_BYTES: + raise FileError(f"Files must be under {MAX_UPLOAD_BYTES // (1024 * 1024)} MB.") + + if _detect_image(payload) is not None: + return _process_image(payload) + if _looks_like_pdf(payload): + return _process_pdf(payload) + return _process_text(payload, filename) + + +# --- Public API -------------------------------------------------------------- +def safe_display_name(filename: str) -> str: + """A filename fit to show. Never used as a path; the stored name is random.""" + cleaned = Path(filename or "file").name.strip() or "file" + return cleaned[:300] + + +def store( + db: DBSession, + *, + user_id: str, + chat_id: str | None, + payload: bytes, + filename: str, +) -> Attachment: + """Process and persist an upload. Raises FileError if it is unusable.""" + prepared = prepare(payload, filename) + + stored_name = f"{secrets.token_hex(16)}{prepared.extension}" + (attachments_dir() / stored_name).write_bytes(prepared.payload) + + attachment = Attachment( + user_id=user_id, + chat_id=chat_id, + filename=safe_display_name(filename), + stored_name=stored_name, + media_type=prepared.media_type, + size_bytes=len(prepared.payload), + kind=prepared.kind, + width=prepared.width, + height=prepared.height, + extracted_text=prepared.extracted_text, + pages=prepared.pages, + truncated=prepared.truncated, + extraction_error=prepared.extraction_error, + ) + db.add(attachment) + db.commit() + log.info( + "stored %s (%s, %d bytes) for user %s", + attachment.filename, + attachment.kind, + attachment.size_bytes, + user_id, + ) + return attachment + + +def delete(db: DBSession, attachment: Attachment) -> None: + path = stored_path(attachment.stored_name) + if path is not None: + path.unlink(missing_ok=True) + db.delete(attachment) + db.commit() + + +def claim(db: DBSession, *, ids: list[str], user_id: str, message_id: str) -> list[Attachment]: + """Bind pending uploads to the message that was just sent. + + Only unclaimed attachments belonging to this user are taken, so a stray or + forged id cannot pull someone else's file into a conversation. + """ + if not ids: + return [] + + pending = list( + db.scalars( + select(Attachment).where( + Attachment.id.in_(ids), + Attachment.user_id == user_id, + Attachment.message_id.is_(None), + ) + ) + ) + for attachment in pending: + attachment.message_id = message_id + db.commit() + return pending + + +def sweep_orphans(db: DBSession, older_than: timedelta = ORPHAN_AGE) -> int: + """Delete uploads that were never attached to a message. + + A file picked in the composer and then abandoned would otherwise sit on + disk forever. + """ + cutoff = datetime.now(UTC) - older_than + orphans = list(db.scalars(select(Attachment).where(Attachment.message_id.is_(None)))) + + removed = 0 + for attachment in orphans: + created = attachment.created_at + if created.tzinfo is None: + created = created.replace(tzinfo=UTC) + if created >= cutoff: + continue + path = stored_path(attachment.stored_name) + if path is not None: + path.unlink(missing_ok=True) + db.delete(attachment) + removed += 1 + + if removed: + db.commit() + log.info("swept %d orphaned upload(s)", removed) + return removed + + +def data_uri(attachment: Attachment) -> str | None: + """Base64 data URI for an image, as sent to a vision model. + + A data URI rather than a link back to this server: a local endpoint has no + route to LLeMbas, and a hosted one has no credentials for it. + """ + import base64 + + path = stored_path(attachment.stored_name) + if path is None: + return None + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + return f"data:{attachment.media_type};base64,{encoded}" diff --git a/src/lembas/web/static/css/chat.css b/src/lembas/web/static/css/chat.css index e63c64b..cb5e54e 100644 --- a/src/lembas/web/static/css/chat.css +++ b/src/lembas/web/static/css/chat.css @@ -412,6 +412,102 @@ } .chat-settings__params .field { margin-bottom: 0; } +/* --- Attachment chips (composer) ------------------------------------------ */ +.composer { position: relative; } + +.composer__attachments { + max-width: var(--thread-max-width); + margin: 0 auto var(--sp-2); + display: flex; + flex-wrap: wrap; + gap: var(--sp-2); +} +.composer__attachments:empty { display: none; } + +.chip { + display: flex; + align-items: center; + gap: var(--sp-2); + padding: var(--sp-2); + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--surface); + max-width: 20rem; + font-size: var(--text-sm); +} +.chip--error { border-color: var(--danger); background: var(--danger-soft); } +.chip--error .chip__icon { color: var(--danger); } + +.chip__thumb { + width: 2.25rem; + height: 2.25rem; + border-radius: var(--radius-sm); + object-fit: cover; + flex: none; +} +.chip__icon { color: var(--ink-muted); flex: none; display: flex; } +.chip__body { min-width: 0; flex: 1; display: flex; flex-direction: column; } +.chip__name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.chip__meta { font-size: var(--text-xs); color: var(--ink-faint); } +.chip__warning { font-size: var(--text-xs); color: var(--danger); } + +.composer__attach { flex: none; align-self: flex-end; } + +/* --- Drag and drop -------------------------------------------------------- */ +.dropzone-overlay { + position: absolute; + inset: 0; + z-index: 5; + display: none; + flex-direction: column; + align-items: center; + justify-content: center; + gap: var(--sp-2); + border: 2px dashed var(--accent); + border-radius: var(--radius-lg); + background: color-mix(in srgb, var(--bg) 88%, var(--accent)); + color: var(--accent); + font-weight: 500; + /* The overlay must not eat the drop event it is advertising. */ + pointer-events: none; +} +.composer.is-dropping .dropzone-overlay { display: flex; } + +/* --- Attachments in the thread -------------------------------------------- */ +.attachments { + display: flex; + flex-wrap: wrap; + gap: var(--sp-2); + margin-bottom: var(--sp-2); +} +.attachments__image { + display: block; + border-radius: var(--radius); + overflow: hidden; + border: 1px solid var(--border); + line-height: 0; +} +.attachments__image img { + max-width: min(22rem, 100%); + max-height: 20rem; + width: auto; + height: auto; + object-fit: contain; +} +.attachments__doc { + display: flex; + align-items: flex-start; + gap: var(--sp-2); + padding: var(--sp-2) var(--sp-3); + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--surface); + font-size: var(--text-sm); + max-width: 100%; +} +.attachments__doc-body { display: flex; flex-direction: column; min-width: 0; } +.attachments__doc-body > a { overflow-wrap: anywhere; } + /* --- Model avatars -------------------------------------------------------- */ .model-avatar { width: 2rem; diff --git a/src/lembas/web/static/js/app.js b/src/lembas/web/static/js/app.js index 81ed765..ae2aa45 100644 --- a/src/lembas/web/static/js/app.js +++ b/src/lembas/web/static/js/app.js @@ -101,12 +101,100 @@ } } + /* --- Attachments ------------------------------------------------------- + Files are uploaded one at a time as soon as they are chosen, dropped or + pasted, rather than all at once when the message is sent. The chip (or the + rejection) then appears immediately, and a large file cannot make the send + button appear to hang. */ + function uploadFiles(fileList) { + var form = document.getElementById("upload-form"); + var target = document.getElementById("attachments"); + if (!form || !target || !fileList || !fileList.length) return; + + Array.prototype.forEach.call(fileList, function (file) { + var body = new FormData(); + body.append("file", file, file.name); + + fetch(form.getAttribute("hx-post"), { method: "POST", body: body }) + .then(function (response) { return response.text(); }) + .then(function (html) { + target.insertAdjacentHTML("beforeend", html); + // The chip's remove button is htmx-driven, so the new markup has to + // be announced or its attributes are inert. + if (window.htmx) window.htmx.process(target.lastElementChild); + }) + .catch(function () { + target.insertAdjacentHTML( + "beforeend", + '
' + + '' + + 'Upload failed.
' + ); + // Set as text, never as HTML: the filename comes from the user. + target.lastElementChild.querySelector(".chip__name").textContent = file.name; + }); + }); + } + + function setupDropzone() { + var zone = document.querySelector("[data-dropzone]"); + if (!zone) return; + + /* dragenter/dragleave fire for every child element the pointer crosses, so + a plain toggle flickers. Counting entries and exits is the standard fix. */ + var depth = 0; + + function hasFiles(event) { + return event.dataTransfer && Array.prototype.indexOf.call( + event.dataTransfer.types || [], "Files" + ) !== -1; + } + + zone.addEventListener("dragenter", function (event) { + if (!hasFiles(event)) return; + event.preventDefault(); + depth += 1; + zone.classList.add("is-dropping"); + }); + + zone.addEventListener("dragover", function (event) { + if (hasFiles(event)) event.preventDefault(); + }); + + zone.addEventListener("dragleave", function () { + depth = Math.max(0, depth - 1); + if (depth === 0) zone.classList.remove("is-dropping"); + }); + + zone.addEventListener("drop", function (event) { + if (!hasFiles(event)) return; + event.preventDefault(); + depth = 0; + zone.classList.remove("is-dropping"); + uploadFiles(event.dataTransfer.files); + }); + + /* Pasting a screenshot straight into the composer. Only files are taken; + pasted text must still behave as text. */ + document.addEventListener("paste", function (event) { + var composer = event.target.closest("[data-composer-input]"); + if (!composer || !event.clipboardData) return; + var files = Array.prototype.filter.call( + event.clipboardData.files || [], function (f) { return f && f.size; } + ); + if (!files.length) return; + event.preventDefault(); + uploadFiles(files); + }); + } + window.lembas = { applyTheme: applyTheme, toggleTheme: toggleTheme, copyText: copyText, scrollThread: scrollThread, - autosize: autosize + autosize: autosize, + uploadFiles: uploadFiles }; /* --- Wiring ------------------------------------------------------------ */ @@ -147,6 +235,7 @@ document.querySelectorAll("[data-autosize]").forEach(autosize); scrollThread(true); applyTheme(currentTheme()); + setupDropzone(); }); /* After any htmx swap: re-measure the composer and follow new content. */ diff --git a/src/lembas/web/templates/chat/_attachment_chip.html b/src/lembas/web/templates/chat/_attachment_chip.html new file mode 100644 index 0000000..1e6042c --- /dev/null +++ b/src/lembas/web/templates/chat/_attachment_chip.html @@ -0,0 +1,38 @@ +{% from "_macros.html" import icon %} +{# + A pending attachment in the composer, before the message is sent. + + Carries its own id in a hidden input so the composer form submits it with the + message; removing the chip removes the input, which is all the bookkeeping + the client needs. +#} +
+ + + {% if attachment.is_image %} + + {% else %} + + {{ icon("attach" if attachment.kind == "document" else "copy", "icon--sm") }} + + {% endif %} + + + {{ attachment.filename }} + + {{ attachment.human_size }} + {%- if attachment.pages %} · {{ attachment.pages }} page{{ '' if attachment.pages == 1 else 's' }}{% endif %} + {%- if attachment.width %} · {{ attachment.width }}×{{ attachment.height }}{% endif %} + {%- if attachment.truncated %} · truncated{% endif %} + + {% if attachment.extraction_error %} + {{ attachment.extraction_error }} + {% endif %} + + + +
diff --git a/src/lembas/web/templates/chat/_attachment_error.html b/src/lembas/web/templates/chat/_attachment_error.html new file mode 100644 index 0000000..d2d0466 --- /dev/null +++ b/src/lembas/web/templates/chat/_attachment_error.html @@ -0,0 +1,17 @@ +{% from "_macros.html" import icon %} +{# + A rejected upload. Rendered in place of a chip so the reason is visible in + the composer rather than only in the network tab. Dismissed by hand; it + carries no hidden input, so it cannot be submitted with the message. +#} +
+ {{ icon("warning", "icon--sm") }} + + {{ filename }} + {{ error }} + + +
diff --git a/src/lembas/web/templates/chat/_message.html b/src/lembas/web/templates/chat/_message.html index a9f8673..090fb46 100644 --- a/src/lembas/web/templates/chat/_message.html +++ b/src/lembas/web/templates/chat/_message.html @@ -38,6 +38,41 @@ {% endif %} + {% if message.attachments %} + {# Above the text, matching the order they were added and the order the + model receives them. #} +
+ {% for attachment in message.attachments %} + {% if attachment.is_image %} + + {{ attachment.filename }} + + {% else %} +
+ {{ icon("attach", "icon--sm") }} + + {{ attachment.filename }} + + {{ attachment.human_size }} + {%- if attachment.pages %} · {{ attachment.pages }} page{{ '' if attachment.pages == 1 else 's' }}{% endif %} + {%- if attachment.truncated %} · truncated{% endif %} + {%- if attachment.extracted_text %} + · view extracted text + {%- endif %} + + {% if attachment.extraction_error %} + {{ attachment.extraction_error }} + {% endif %} + +
+ {% endif %} + {% endfor %} +
+ {% endif %} + {% if streaming %} {# Reasoning arrives before the answer, so this block sits above it. It starts open (watching a model think is the point) and the :has() rule @@ -92,9 +127,11 @@ {% endif %} {% elif message.role == "assistant" %}
{{ body_html|safe }}
- {% else %} + {% elif message.content %}
{{ message.content }}
{% endif %} + {# An attachment-only turn has no text; rendering the bubble anyway would + leave an empty box under the file. #} {% if not streaming %}