From bdce2764b1ae5f7d00f07a65f8b85c74e3aa2a0d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Bene=C5=A1?= Date: Tue, 21 Jul 2026 12:19:59 +0200 Subject: [PATCH] File attachments: images for vision, PDFs and text into the prompt Drag, paste or pick a file in the composer. Images go to vision models as multimodal content parts; PDFs and text files have their content extracted and placed in the prompt. Verified end to end against gemma4-e4b-q8 on llama-swap: given a drawing and a text file, it named the red square and blue circle and read the number out of the document. Type is decided by inspecting the bytes, never the filename or the browser's Content-Type -- a .png full of text is stored as text. Images are downscaled to 1400px and re-encoded: a phone photo is several megabytes of base64, which is slow and a large slice of the context window. PDF text is extracted once, at upload, and stored; re-extracting per request would let a reply change because a parser was upgraded. Design points worth keeping: - Images are only sent to models an administrator has marked `vision`. This is not graceful degradation -- most endpoints reject the entire request rather than ignoring an image part. A plain text turn stays a plain string for the same reason: the list form 400s on endpoints that do not implement it. - Images reach the model as base64 data URIs, not links. A local endpoint has no route back to LLeMbas, and a hosted one has no credentials for it. - Non-images are served Content-Disposition: attachment with nosniff, so an uploaded .html can never execute in this origin. Stored names are random; the uploader's name is a label and never a path. - Uploads are unbound until the message is sent, which is what lets a file be removed beforehand. claim() only takes unclaimed rows owned by the sender, so a forged id cannot pull in someone else's file. Abandoned uploads are swept at startup. - A scanned PDF says so rather than silently contributing nothing, and truncation is declared to the model in the document tag so it can admit it did not see page 400. - "Here, look at this" with no words is a legitimate turn, so a message is only empty when it carries neither text nor files. Also fixes auto-titling, which read message["content"] as a string and would have broken on the first multimodal turn. 186 tests, ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- CLAUDE.md | 36 +- README.md | 7 +- pyproject.toml | 2 + src/lembas/api/chats.py | 32 +- src/lembas/api/files.py | 117 +++++ src/lembas/db/models/__init__.py | 10 + src/lembas/db/models/attachment.py | 74 +++ src/lembas/db/models/chat.py | 13 + src/lembas/main.py | 13 + src/lembas/security/permissions.py | 8 + src/lembas/services/chat.py | 86 ++- src/lembas/services/files.py | 400 ++++++++++++++ src/lembas/web/static/css/chat.css | 96 ++++ src/lembas/web/static/js/app.js | 91 +++- .../web/templates/chat/_attachment_chip.html | 38 ++ .../web/templates/chat/_attachment_error.html | 17 + src/lembas/web/templates/chat/_message.html | 39 +- src/lembas/web/templates/chat/index.html | 46 +- tests/test_files.py | 496 ++++++++++++++++++ 19 files changed, 1601 insertions(+), 20 deletions(-) create mode 100644 src/lembas/api/files.py create mode 100644 src/lembas/db/models/attachment.py create mode 100644 src/lembas/services/files.py create mode 100644 src/lembas/web/templates/chat/_attachment_chip.html create mode 100644 src/lembas/web/templates/chat/_attachment_error.html create mode 100644 tests/test_files.py diff --git a/CLAUDE.md b/CLAUDE.md index 9062c09..2118f17 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ lembas info # paths + counts, useful when confused lembas secret-key # generate LEMBAS_SECRET_KEY lembas create-admin # create or promote an admin -pytest # 143 tests, ~5s +pytest # 186 tests, ~7s ruff check . # lint (line length 100) python scripts/build_artwork.py # regenerate all SVG artwork python scripts/fetch_vendor.py # verify vendored JS against the lockfile @@ -75,6 +75,7 @@ src/lembas/ admin.py connections + instance settings admin_models.py model ordering, defaults, images, access admin_users.py users, groups, permissions + files.py upload, serve, remove attachments preferences.py per-user theme, default model, password db/ base.py Base, UUID/Timestamp mixins @@ -87,6 +88,7 @@ src/lembas/ chat.py request building, endpoint resolution, titles markdown.py markdown-it + pygments + nh3 crypto.py Fernet encrypt/decrypt/mask + files.py attachment validation, images, PDF/text extraction reasoning.py splits thinking from the answer settings_store.py runtime instance settings uploads.py validated image storage @@ -167,6 +169,27 @@ llama-swap, vLLM) or `` tags inline in `content` (Ollama and friends). the tags arrive split across chunks. Reasoning is stored in `Message.reasoning` and is deliberately **not** replayed as context on the next turn. +**Attachments are typed by their bytes, not their name.** `services/files.py` +sniffs magic numbers; a `.png` full of text is stored as text. Images are +downscaled and re-encoded (a phone photo is megabytes of base64), PDFs have +their text extracted **once at upload** — re-extracting per request would let a +reply change because a parser was upgraded. + +**Images only go to models marked `vision`.** Sending content parts to an +endpoint without multimodal support is not graceful degradation; most reject +the whole request. `build_request()` checks the capability and falls back to a +plain string. A plain text turn must *stay* a plain string for the same reason. + +**Attachments are served, never linked.** Images reach the model as base64 data +URIs: a local endpoint has no route back to LLeMbas and a hosted one has no +credentials. Non-images are served `Content-Disposition: attachment` with +`nosniff`, so an uploaded `.html` cannot execute in this origin. + +**Uploads are unbound until the message is sent.** `Attachment.message_id` is +null in the composer; `files.claim()` binds them, and only unclaimed rows owned +by that user, so a forged id cannot pull in someone else's file. Abandoned ones +are swept at startup. + **JSON columns need reassignment.** `user.settings_json["theme"] = x` on a plain dict is not detected. The columns use `MutableDict` (`db/types.py`), but the safe habit is `obj.field = {**obj.field, "k": v}`. @@ -215,9 +238,10 @@ notes describe the machine. ## Not built yet -File upload / vision / PDFs, built-in tools + admin tool settings, custom tools -and MCP, agentic execution (local subprocess and SSH connection profiles), image -generation. Nav entries mark where each one goes. +Built-in tools + admin tool settings, custom tools and MCP, agentic execution +(local subprocess and SSH connection profiles), image generation. Nav entries +mark where each one goes. -`Model.capabilities_json` already carries `vision` and `tools` flags that -nothing reads yet — they are admin overrides waiting for those features. +`Model.capabilities_json` carries a `tools` flag nothing reads yet. No OCR: +a scanned PDF is stored with an explanatory `extraction_error` rather than +silently contributing nothing. diff --git a/README.md b/README.md index 06641af..95b4d92 100644 --- a/README.md +++ b/README.md @@ -36,6 +36,9 @@ runtime. Clone it, `pip install -e .`, run it. - **Reasoning display** — thinking from reasoning models streams into its own collapsible block, labelled with how long it took, and is never replayed as context +- **Attachments** — drag, paste or pick images, PDFs and text files. Images are + downscaled and sent to vision models; PDF and text content is extracted and + put in the prompt - **Folders** — arbitrarily nested, delete a folder without losing the chats inside it - **OpenAI connections** — point at OpenAI, LM Studio, vLLM, llama.cpp, @@ -53,8 +56,8 @@ runtime. Clone it, `pip install -e .`, run it. **Planned** -File upload, vision and PDFs · built-in tools with admin settings · custom tools -and MCP servers · agentic execution (local and over SSH) · image generation. +Built-in tools with admin settings · custom tools and MCP servers · agentic +execution (local and over SSH) · image generation · OCR for scanned PDFs. ## Quick start diff --git a/pyproject.toml b/pyproject.toml index ba6dd0b..0b28960 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,6 +33,8 @@ dependencies = [ "linkify-it-py>=2.0", # bare URLs in model output become links "pygments>=2.18", "nh3>=0.2.18", + "pypdf>=5.1", # PDF text extraction for attachments + "pillow>=11.0", # image validation and downscaling for vision "typer>=0.12", ] diff --git a/src/lembas/api/chats.py b/src/lembas/api/chats.py index c5a941c..a825183 100644 --- a/src/lembas/api/chats.py +++ b/src/lembas/api/chats.py @@ -16,6 +16,7 @@ from lembas.db.models import ROLE_ASSISTANT, ROLE_USER, Chat, Message, User from lembas.db.session import session_scope from lembas.security import permissions from lembas.services import chat as chat_service +from lembas.services import files as files_service from lembas.services import sse from lembas.services.llm.openai_client import ( LLMError, @@ -66,7 +67,8 @@ async def post_message( db: Db, user: RequiredUser, chat_id: str, - content: str = Form(...), + content: str = Form(""), + file_ids: list[str] = Form(default=[]), ) -> Response: """Persist the user's turn and hand back the pair of bubbles. @@ -77,10 +79,15 @@ async def post_message( chat = _owned_chat(db, chat_id, user.id) content = content.strip() - if not content: + # "Here, look at this" with no words is a legitimate turn, so an empty + # message is only empty when it carries nothing at all. + if not content and not file_ids: return Response(status_code=status.HTTP_204_NO_CONTENT) user_message = chat_service.create_message(db, chat, ROLE_USER, content) + if file_ids: + files_service.claim(db, ids=file_ids, user_id=user.id, message_id=user_message.id) + db.refresh(user_message) assistant_message = chat_service.create_message( db, chat, ROLE_ASSISTANT, "", complete_=False, model_id=chat.model_id ) @@ -130,6 +137,19 @@ async def stream_message( ) +def _plain_text(content: str | list) -> str: + """The text of a message payload, whether it is a string or content parts.""" + if isinstance(content, str): + return content + if isinstance(content, list): + return " ".join( + part.get("text", "") + for part in content + if isinstance(part, dict) and part.get("type") == "text" + ).strip() + return "" + + async def _generate(chat_id: str, message_id: str) -> AsyncIterator[str]: """Drive one completion and frame it as SSE. @@ -159,8 +179,14 @@ async def _generate(chat_id: str, message_id: str) -> AsyncIterator[str]: try: endpoint, model_id = chat_service.resolve_endpoint(db, chat) payload = chat_service.build_request(db, chat, upto=message) + # A multimodal turn's content is a list of parts, not a string, so + # the text has to be picked out before it can title a chat. first_user_text = next( - (m["content"] for m in reversed(payload["messages"]) if m["role"] == ROLE_USER), + ( + _plain_text(m["content"]) + for m in reversed(payload["messages"]) + if m["role"] == ROLE_USER + ), "", ) diff --git a/src/lembas/api/files.py b/src/lembas/api/files.py new file mode 100644 index 0000000..5d6e22f --- /dev/null +++ b/src/lembas/api/files.py @@ -0,0 +1,117 @@ +"""Uploading, serving and removing chat attachments.""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status +from fastapi.responses import FileResponse + +from lembas.api.deps import Db, RequiredUser, require_permission +from lembas.db.models import Attachment +from lembas.services import files as files_service +from lembas.web.templating import templates + +log = logging.getLogger(__name__) + +router = APIRouter(prefix="/api/files", tags=["files"]) + + +def _owned(db: Db, attachment_id: str, user_id: str) -> Attachment: + attachment = db.get(Attachment, attachment_id) + if attachment is None or attachment.user_id != user_id: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That file no longer exists.") + return attachment + + +@router.post("", dependencies=[Depends(require_permission("files.upload"))]) +async def upload( + request: Request, + db: Db, + user: RequiredUser, + file: UploadFile = File(...), + chat_id: str = "", +) -> Response: + """Accept one file and return the chip that represents it in the composer. + + The attachment is stored immediately but left unbound: it only joins a + message when that message is sent. That is what lets a file be removed + before sending, and what the orphan sweep later cleans up. + """ + payload = await file.read() + + try: + attachment = files_service.store( + db, + user_id=user.id, + chat_id=chat_id or None, + payload=payload, + filename=file.filename or "file", + ) + except files_service.FileError as exc: + # 200 with an error chip rather than a 4xx: htmx swaps the response + # body either way, and an error the user can read beats a silent + # failure in the console. + return templates.TemplateResponse( + request, + "chat/_attachment_error.html", + {"request": request, "filename": file.filename or "file", "error": str(exc)}, + ) + + return templates.TemplateResponse( + request, + "chat/_attachment_chip.html", + {"request": request, "attachment": attachment}, + ) + + +@router.delete("/{attachment_id}") +async def remove(db: Db, user: RequiredUser, attachment_id: str) -> Response: + """Detach a file before it has been sent.""" + attachment = _owned(db, attachment_id, user.id) + if attachment.message_id is not None: + # Deleting it now would rewrite a conversation that has already been + # sent to a model and read by the user. + raise HTTPException( + status.HTTP_409_CONFLICT, "That file is part of a sent message." + ) + files_service.delete(db, attachment) + return Response(status_code=status.HTTP_200_OK) + + +@router.get("/{attachment_id}/content") +async def content(db: Db, user: RequiredUser, attachment_id: str) -> Response: + """Serve an attachment back to its owner.""" + attachment = _owned(db, attachment_id, user.id) + path = files_service.stored_path(attachment.stored_name) + if path is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That file is no longer on disk.") + + # inline for images so they render in the thread; attachment for everything + # else so a text/html upload can never be executed in this origin. + disposition = "inline" if attachment.is_image else "attachment" + return FileResponse( + path, + media_type=attachment.media_type if attachment.is_image else "application/octet-stream", + headers={ + "Content-Disposition": f'{disposition}; filename="{attachment.filename}"', + "Cache-Control": "private, max-age=604800", + # Belt and braces: even for images, never let a browser sniff its + # way to treating the bytes as something executable. + "X-Content-Type-Options": "nosniff", + }, + ) + + +@router.get("/{attachment_id}/text") +async def extracted_text(db: Db, user: RequiredUser, attachment_id: str) -> Response: + """The text a document contributed to the prompt. + + Worth being able to see: a PDF that extracted badly explains a strange + reply, and there is otherwise no way to tell what the model was given. + """ + attachment = _owned(db, attachment_id, user.id) + return Response( + attachment.extracted_text or attachment.extraction_error, + media_type="text/plain; charset=utf-8", + ) diff --git a/src/lembas/db/models/__init__.py b/src/lembas/db/models/__init__.py index d6f51ec..4a8e506 100644 --- a/src/lembas/db/models/__init__.py +++ b/src/lembas/db/models/__init__.py @@ -5,6 +5,12 @@ what ``init_db()`` relies on to create the schema at startup. Any new model module must be imported here or its table will silently never be created. """ +from lembas.db.models.attachment import ( + KIND_DOCUMENT, + KIND_IMAGE, + KIND_TEXT, + Attachment, +) from lembas.db.models.chat import ( ROLE_ASSISTANT, ROLE_SYSTEM, @@ -26,6 +32,10 @@ from lembas.db.models.user import ( ) __all__ = [ + "Attachment", + "KIND_DOCUMENT", + "KIND_IMAGE", + "KIND_TEXT", "ROLE_ADMIN", "ROLE_ASSISTANT", "ROLE_PENDING", diff --git a/src/lembas/db/models/attachment.py b/src/lembas/db/models/attachment.py new file mode 100644 index 0000000..af7bc99 --- /dev/null +++ b/src/lembas/db/models/attachment.py @@ -0,0 +1,74 @@ +"""Files attached to chat messages.""" + +from __future__ import annotations + +from sqlalchemy import Boolean, ForeignKey, Integer, String, Text +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from lembas.db.base import Base, Timestamps, UUIDPrimaryKey + +# What the file is for, decided at upload time. Drives both how it is rendered +# and how it reaches the model: images become multimodal parts, everything else +# becomes text in the prompt. +KIND_IMAGE = "image" +KIND_DOCUMENT = "document" # PDF: text is extracted +KIND_TEXT = "text" # plain text, markdown, csv, source code + + +class Attachment(UUIDPrimaryKey, Timestamps, Base): + __tablename__ = "attachments" + + user_id: Mapped[str] = mapped_column( + String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + chat_id: Mapped[str | None] = mapped_column( + String(32), ForeignKey("chats.id", ondelete="CASCADE"), index=True + ) + # Null while the file is uploaded but the message has not been sent yet. + # Those orphans are swept periodically -- see services.files.sweep_orphans. + message_id: Mapped[str | None] = mapped_column( + String(32), ForeignKey("messages.id", ondelete="CASCADE"), index=True + ) + + # What the uploader called it. Display only, never used as a path. + filename: Mapped[str] = mapped_column(String(300), nullable=False) + # Random name on disk. See services.files for why the two are separate. + stored_name: Mapped[str] = mapped_column(String(120), nullable=False) + + media_type: Mapped[str] = mapped_column(String(100), default="") + size_bytes: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + kind: Mapped[str] = mapped_column(String(16), default=KIND_DOCUMENT, nullable=False) + + # Images only, after downscaling. + width: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + height: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + + # Documents and text: the content that actually reaches the model. Held in + # the database rather than re-extracted per request -- extraction is slow, + # and a reply must not silently change because a PDF parser was upgraded. + extracted_text: Mapped[str] = mapped_column(Text, default="") + pages: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + truncated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + + # Non-empty when the file was stored but its text could not be read, e.g. a + # scanned PDF with no text layer. Shown next to the attachment so the user + # is not left wondering why the model ignored it. + extraction_error: Mapped[str] = mapped_column(Text, default="") + + message: Mapped[Message] = relationship(back_populates="attachments") # noqa: F821 + + @property + def is_image(self) -> bool: + return self.kind == KIND_IMAGE + + @property + def human_size(self) -> str: + size = float(self.size_bytes) + for unit in ("B", "KB", "MB"): + if size < 1024 or unit == "MB": + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} MB" + + def __repr__(self) -> str: + return f"" diff --git a/src/lembas/db/models/chat.py b/src/lembas/db/models/chat.py index 9f4f5c2..a44c158 100644 --- a/src/lembas/db/models/chat.py +++ b/src/lembas/db/models/chat.py @@ -120,6 +120,19 @@ class Message(UUIDPrimaryKey, Timestamps, Base): complete: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) chat: Mapped[Chat] = relationship(back_populates="messages") + attachments: Mapped[list[Attachment]] = relationship( # noqa: F821 + back_populates="message", + cascade="all, delete-orphan", + order_by="Attachment.created_at", + ) + + @property + def images(self) -> list: + return [a for a in self.attachments if a.is_image] + + @property + def documents(self) -> list: + return [a for a in self.attachments if not a.is_image] def __repr__(self) -> str: return f"" diff --git a/src/lembas/main.py b/src/lembas/main.py index b3e694d..cf3d7d5 100644 --- a/src/lembas/main.py +++ b/src/lembas/main.py @@ -18,6 +18,7 @@ from lembas.api import ( admin_users, auth, chats, + files, folders, pages, preferences, @@ -52,6 +53,17 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: ' python -c "import secrets; print(secrets.token_urlsafe(48))"' ) + # Files chosen in a composer that was never sent would otherwise sit on + # disk forever. Cheap, and startup is the natural moment for it. + try: + from lembas.db.session import session_scope + from lembas.services.files import sweep_orphans + + with session_scope() as db: + sweep_orphans(db) + except Exception: # noqa: BLE001 - housekeeping must never block startup + log.exception("orphaned upload sweep failed") + log.info("LLeMbas %s starting on http://%s:%s", __version__, settings.host, settings.port) log.info("data directory: %s", settings.data_dir.resolve()) yield @@ -74,6 +86,7 @@ def create_app() -> FastAPI: app.include_router(auth.router) app.include_router(preferences.router) app.include_router(chats.router) + app.include_router(files.router) app.include_router(folders.router) app.include_router(admin.router) app.include_router(admin_users.router) diff --git a/src/lembas/security/permissions.py b/src/lembas/security/permissions.py index f3385dd..8ec2544 100644 --- a/src/lembas/security/permissions.py +++ b/src/lembas/security/permissions.py @@ -70,6 +70,14 @@ PERMISSION_DEFS: tuple[PermissionDef, ...] = ( True, "Workspace", ), + PermissionDef( + "files.upload", + "Attach files", + "Attach images, PDFs and text files to a message. Images only reach " + "models marked as having vision.", + True, + "Workspace", + ), ) PERMISSION_KEYS = tuple(d.key for d in PERMISSION_DEFS) diff --git a/src/lembas/services/chat.py b/src/lembas/services/chat.py index e7151de..5eb66f0 100644 --- a/src/lembas/services/chat.py +++ b/src/lembas/services/chat.py @@ -17,6 +17,7 @@ from lembas.db.models import ( Message, Model, ) +from lembas.services import files as files_service from lembas.services.llm.openai_client import Endpoint, LLMError, complete log = logging.getLogger(__name__) @@ -71,7 +72,66 @@ def resolve_endpoint(db: DBSession, chat: Chat) -> tuple[Endpoint, str]: return Endpoint.from_connection(connection), chat.model_id -def build_messages(db: DBSession, chat: Chat, *, upto: Message | None = None) -> list[dict]: +def document_context(message: Message) -> str: + """Extracted text from a message's non-image attachments. + + Wrapped in named tags so the model can tell one document from another, and + tell all of them from what the user actually typed. Truncation is stated + inline rather than silently, so a model asked about page 400 of a 300-page + extract can say it did not see it. + """ + blocks: list[str] = [] + for attachment in message.documents: + if not attachment.extracted_text.strip(): + continue + note = " (truncated)" if attachment.truncated else "" + blocks.append( + f'\n' + f"{attachment.extracted_text.strip()}\n" + f"" + ) + return "\n\n".join(blocks) + + +def message_payload(message: Message, *, vision: bool) -> dict[str, Any]: + """One history entry in the shape the endpoint expects. + + Plain text stays a plain string: sending the multimodal list form to an + endpoint that does not implement it is a reliable way to get a 400, and + most local runners do not. + """ + text = message.content.strip() + + documents = document_context(message) + if documents: + # Documents lead so the question that follows has its material already + # in view, which is how these models are trained to read a prompt. + text = f"{documents}\n\n{text}" if text else documents + + images = message.images if vision else [] + if not images: + return {"role": message.role, "content": text} + + parts: list[dict[str, Any]] = [] + if text: + parts.append({"type": "text", "text": text}) + for attachment in images: + uri = files_service.data_uri(attachment) + if uri is None: + # The row survived but the file did not. Better to say so than to + # send a turn that silently lost its picture. + log.warning("attachment %s has no file on disk", attachment.id) + continue + parts.append({"type": "image_url", "image_url": {"url": uri}}) + + if not parts: + return {"role": message.role, "content": text} + return {"role": message.role, "content": parts} + + +def build_messages( + db: DBSession, chat: Chat, *, upto: Message | None = None, vision: bool = False +) -> list[dict]: """Assemble the message list to send upstream. `upto` excludes the placeholder assistant row being generated into, and @@ -88,24 +148,38 @@ def build_messages(db: DBSession, chat: Chat, *, upto: Message | None = None) -> for message in history: if upto is not None and message.id == upto.id: break - # Skip turns that failed or produced nothing: sending an empty - # assistant message upsets several providers. - if message.error or not message.content.strip(): + # Skip turns that failed or produced nothing -- but a message carrying + # only an attachment has no text and must still be sent. + if message.error: continue - payload.append({"role": message.role, "content": message.content}) + if not message.content.strip() and not message.attachments: + continue + payload.append(message_payload(message, vision=vision)) return payload +def model_supports(db: DBSession, chat: Chat, capability: str) -> bool: + """Whether the chat's current model is marked as having a capability.""" + model = db.scalar( + select(Model).where(Model.model_id == chat.model_id).order_by(Model.position) + ) + return bool(model and (model.capabilities_json or {}).get(capability)) + + def build_request(db: DBSession, chat: Chat, *, upto: Message | None = None) -> dict[str, Any]: params = { key: value for key, value in (chat.params_json or {}).items() if key in FORWARDED_PARAMS and value not in (None, "") } + # Images are only sent to a model an administrator has marked as having + # vision. Sending them to one that has not is not a graceful degradation: + # most endpoints reject the whole request. + vision = model_supports(db, chat, "vision") return { "model": chat.model_id, - "messages": build_messages(db, chat, upto=upto), + "messages": build_messages(db, chat, upto=upto, vision=vision), **params, } diff --git a/src/lembas/services/files.py b/src/lembas/services/files.py new file mode 100644 index 0000000..06eb6d8 --- /dev/null +++ b/src/lembas/services/files.py @@ -0,0 +1,400 @@ +"""Storing and reading uploaded attachments. + +Three kinds of file, each handled differently on the way to the model: + +* **Images** are downscaled and re-encoded, then sent as multimodal content + parts. Downscaling is not cosmetic -- a phone photo is several megabytes of + base64, which is both slow and a large slice of the context window. +* **PDFs** have their text extracted once, at upload. Extraction is slow and a + reply must not silently change because a parser was upgraded later. +* **Plain text** (including source code and CSV) is decoded and stored as-is. + +Everything an uploader supplies is treated as hostile: the type is decided by +inspecting the bytes rather than trusting the browser, the name on disk is +random, and both image dimensions and PDF page counts are capped so a small +file cannot expand into an enormous amount of work. +""" + +from __future__ import annotations + +import io +import logging +import secrets +from dataclasses import dataclass +from datetime import UTC, datetime, timedelta +from pathlib import Path + +from PIL import Image, UnidentifiedImageError +from sqlalchemy import select +from sqlalchemy.orm import Session as DBSession + +from lembas.config import settings +from lembas.db.models import KIND_DOCUMENT, KIND_IMAGE, KIND_TEXT, Attachment + +log = logging.getLogger(__name__) + +# --- Limits ------------------------------------------------------------------ +MAX_UPLOAD_BYTES = 20 * 1024 * 1024 + +# Longest edge after downscaling. Large enough for a model to read a screenshot +# or a page of text, small enough that the base64 stays reasonable. +MAX_IMAGE_EDGE = 1400 +JPEG_QUALITY = 85 + +# Pillow's own guard against decompression bombs: a 60,000x60,000 PNG is a few +# KB on disk and hundreds of GB decoded. +Image.MAX_IMAGE_PIXELS = 64_000_000 + +MAX_PDF_PAGES = 300 +# Characters of extracted text kept per document. Roughly 30k tokens, which is +# already a large slice of most context windows; more is rarely useful and +# frequently breaks the request outright. +MAX_EXTRACTED_CHARS = 120_000 + +# Orphans are files uploaded into a composer that was never sent. +ORPHAN_AGE = timedelta(hours=24) + +IMAGE_TYPES: dict[bytes, tuple[str, str]] = { + b"\x89PNG\r\n\x1a\n": ("image/png", ".png"), + b"\xff\xd8\xff": ("image/jpeg", ".jpg"), + b"GIF87a": ("image/gif", ".gif"), + b"GIF89a": ("image/gif", ".gif"), +} + +# Extensions treated as text when the bytes decode cleanly as UTF-8. The list +# exists only to pick a sensible media type; decodability is what actually +# decides, so an unlisted extension still works. +TEXT_EXTENSIONS = { + ".txt": "text/plain", ".md": "text/markdown", ".markdown": "text/markdown", + ".csv": "text/csv", ".tsv": "text/tab-separated-values", + ".json": "application/json", ".yaml": "text/yaml", ".yml": "text/yaml", + ".toml": "text/toml", ".ini": "text/plain", ".cfg": "text/plain", + ".xml": "text/xml", ".html": "text/plain", ".css": "text/plain", + ".py": "text/x-python", ".js": "text/javascript", ".ts": "text/typescript", + ".rs": "text/x-rust", ".go": "text/x-go", ".c": "text/x-c", ".h": "text/x-c", + ".cpp": "text/x-c++", ".java": "text/x-java", ".rb": "text/x-ruby", + ".sh": "text/x-shellscript", ".sql": "text/x-sql", ".log": "text/plain", +} + + +class FileError(Exception): + """A rejected upload, with a message fit to show the user.""" + + +@dataclass +class Prepared: + """The result of inspecting and processing an upload, before it is stored.""" + + payload: bytes + kind: str + media_type: str + extension: str + width: int = 0 + height: int = 0 + extracted_text: str = "" + pages: int = 0 + truncated: bool = False + extraction_error: str = "" + + +# --- Storage ----------------------------------------------------------------- +def attachments_dir() -> Path: + path = settings.uploads_dir / "attachments" + path.mkdir(parents=True, exist_ok=True) + return path + + +def stored_path(stored_name: str) -> Path | None: + """Resolve a stored name to a path, refusing anything outside the directory.""" + if not stored_name or "/" in stored_name or "\\" in stored_name or stored_name.startswith("."): + return None + base = attachments_dir().resolve() + path = (base / stored_name).resolve() + try: + path.relative_to(base) + except ValueError: + return None + return path if path.is_file() else None + + +# --- Type detection ---------------------------------------------------------- +def _detect_image(payload: bytes) -> tuple[str, str] | None: + for signature, (media_type, extension) in IMAGE_TYPES.items(): + if payload.startswith(signature): + return media_type, extension + if payload[:4] == b"RIFF" and payload[8:12] == b"WEBP": + return "image/webp", ".webp" + return None + + +def _looks_like_pdf(payload: bytes) -> bool: + # The header is allowed a little leading junk by the spec, and real files + # in the wild use it. + return b"%PDF-" in payload[:1024] + + +# --- Processing -------------------------------------------------------------- +def _process_image(payload: bytes) -> Prepared: + try: + with Image.open(io.BytesIO(payload)) as image: + image.load() + has_alpha = image.mode in ("RGBA", "LA", "P") and "transparency" in image.info + # Animation is lost on re-encode; keeping only the first frame is + # honest and is what a model would see anyway. + frame = image.convert("RGBA" if has_alpha else "RGB") + + width, height = frame.size + longest = max(width, height) + if longest > MAX_IMAGE_EDGE: + scale = MAX_IMAGE_EDGE / longest + frame = frame.resize( + (max(1, int(width * scale)), max(1, int(height * scale))), + Image.LANCZOS, + ) + + buffer = io.BytesIO() + if has_alpha: + frame.save(buffer, format="PNG", optimize=True) + media_type, extension = "image/png", ".png" + else: + frame.save(buffer, format="JPEG", quality=JPEG_QUALITY, optimize=True) + media_type, extension = "image/jpeg", ".jpg" + + return Prepared( + payload=buffer.getvalue(), + kind=KIND_IMAGE, + media_type=media_type, + extension=extension, + width=frame.width, + height=frame.height, + ) + except Image.DecompressionBombError as exc: + raise FileError("That image's dimensions are implausibly large.") from exc + except (UnidentifiedImageError, OSError, ValueError) as exc: + raise FileError("That image could not be read. Is it corrupt?") from exc + + +def _process_pdf(payload: bytes) -> Prepared: + from pypdf import PdfReader + from pypdf.errors import PdfReadError + + prepared = Prepared( + payload=payload, kind=KIND_DOCUMENT, media_type="application/pdf", extension=".pdf" + ) + + try: + reader = PdfReader(io.BytesIO(payload)) + if reader.is_encrypted: + # An empty password unlocks a surprising number of "encrypted" PDFs. + try: + reader.decrypt("") + except Exception: # noqa: BLE001 - any failure means the same thing + prepared.extraction_error = ( + "This PDF is password-protected, so its text could not be read." + ) + return prepared + + prepared.pages = len(reader.pages) + chunks: list[str] = [] + total = 0 + + for index, page in enumerate(reader.pages[:MAX_PDF_PAGES]): + try: + text = page.extract_text() or "" + except Exception as exc: # noqa: BLE001 - one bad page is not fatal + log.debug("page %d of a PDF failed to extract: %s", index, exc) + continue + if not text.strip(): + continue + chunks.append(f"[page {index + 1}]\n{text.strip()}") + total += len(text) + if total >= MAX_EXTRACTED_CHARS: + prepared.truncated = True + break + + if prepared.pages > MAX_PDF_PAGES: + prepared.truncated = True + + prepared.extracted_text = "\n\n".join(chunks)[:MAX_EXTRACTED_CHARS] + + if not prepared.extracted_text.strip(): + # Almost always a scan. Saying so beats the model silently ignoring + # a document the user believes it can read. + prepared.extraction_error = ( + "No text could be extracted. This looks like a scanned PDF; " + "LLeMbas does not do OCR yet." + ) + + except PdfReadError as exc: + prepared.extraction_error = "This file is not a readable PDF." + log.info("unreadable PDF: %s", exc) + except Exception as exc: # noqa: BLE001 - never let a bad file 500 the upload + prepared.extraction_error = "This PDF could not be read." + log.warning("unexpected PDF failure: %s", exc) + + return prepared + + +def _process_text(payload: bytes, filename: str) -> Prepared: + for encoding in ("utf-8", "utf-16", "latin-1"): + try: + text = payload.decode(encoding) + break + except (UnicodeDecodeError, LookupError): + continue + else: + raise FileError("That file is not text, and is not a format LLeMbas can read.") + + # Null bytes mean this decoded by luck (latin-1 decodes any byte) and is + # really a binary file. + if "\x00" in text[:4096]: + raise FileError("That file is not text, and is not a format LLeMbas can read.") + + truncated = len(text) > MAX_EXTRACTED_CHARS + extension = Path(filename).suffix.lower() + + return Prepared( + payload=payload, + kind=KIND_TEXT, + media_type=TEXT_EXTENSIONS.get(extension, "text/plain"), + extension=extension if extension in TEXT_EXTENSIONS else ".txt", + extracted_text=text[:MAX_EXTRACTED_CHARS], + truncated=truncated, + ) + + +def prepare(payload: bytes, filename: str) -> Prepared: + """Inspect an upload, decide what it is, and process it accordingly.""" + if not payload: + raise FileError("That file is empty.") + if len(payload) > MAX_UPLOAD_BYTES: + raise FileError(f"Files must be under {MAX_UPLOAD_BYTES // (1024 * 1024)} MB.") + + if _detect_image(payload) is not None: + return _process_image(payload) + if _looks_like_pdf(payload): + return _process_pdf(payload) + return _process_text(payload, filename) + + +# --- Public API -------------------------------------------------------------- +def safe_display_name(filename: str) -> str: + """A filename fit to show. Never used as a path; the stored name is random.""" + cleaned = Path(filename or "file").name.strip() or "file" + return cleaned[:300] + + +def store( + db: DBSession, + *, + user_id: str, + chat_id: str | None, + payload: bytes, + filename: str, +) -> Attachment: + """Process and persist an upload. Raises FileError if it is unusable.""" + prepared = prepare(payload, filename) + + stored_name = f"{secrets.token_hex(16)}{prepared.extension}" + (attachments_dir() / stored_name).write_bytes(prepared.payload) + + attachment = Attachment( + user_id=user_id, + chat_id=chat_id, + filename=safe_display_name(filename), + stored_name=stored_name, + media_type=prepared.media_type, + size_bytes=len(prepared.payload), + kind=prepared.kind, + width=prepared.width, + height=prepared.height, + extracted_text=prepared.extracted_text, + pages=prepared.pages, + truncated=prepared.truncated, + extraction_error=prepared.extraction_error, + ) + db.add(attachment) + db.commit() + log.info( + "stored %s (%s, %d bytes) for user %s", + attachment.filename, + attachment.kind, + attachment.size_bytes, + user_id, + ) + return attachment + + +def delete(db: DBSession, attachment: Attachment) -> None: + path = stored_path(attachment.stored_name) + if path is not None: + path.unlink(missing_ok=True) + db.delete(attachment) + db.commit() + + +def claim(db: DBSession, *, ids: list[str], user_id: str, message_id: str) -> list[Attachment]: + """Bind pending uploads to the message that was just sent. + + Only unclaimed attachments belonging to this user are taken, so a stray or + forged id cannot pull someone else's file into a conversation. + """ + if not ids: + return [] + + pending = list( + db.scalars( + select(Attachment).where( + Attachment.id.in_(ids), + Attachment.user_id == user_id, + Attachment.message_id.is_(None), + ) + ) + ) + for attachment in pending: + attachment.message_id = message_id + db.commit() + return pending + + +def sweep_orphans(db: DBSession, older_than: timedelta = ORPHAN_AGE) -> int: + """Delete uploads that were never attached to a message. + + A file picked in the composer and then abandoned would otherwise sit on + disk forever. + """ + cutoff = datetime.now(UTC) - older_than + orphans = list(db.scalars(select(Attachment).where(Attachment.message_id.is_(None)))) + + removed = 0 + for attachment in orphans: + created = attachment.created_at + if created.tzinfo is None: + created = created.replace(tzinfo=UTC) + if created >= cutoff: + continue + path = stored_path(attachment.stored_name) + if path is not None: + path.unlink(missing_ok=True) + db.delete(attachment) + removed += 1 + + if removed: + db.commit() + log.info("swept %d orphaned upload(s)", removed) + return removed + + +def data_uri(attachment: Attachment) -> str | None: + """Base64 data URI for an image, as sent to a vision model. + + A data URI rather than a link back to this server: a local endpoint has no + route to LLeMbas, and a hosted one has no credentials for it. + """ + import base64 + + path = stored_path(attachment.stored_name) + if path is None: + return None + encoded = base64.b64encode(path.read_bytes()).decode("ascii") + return f"data:{attachment.media_type};base64,{encoded}" diff --git a/src/lembas/web/static/css/chat.css b/src/lembas/web/static/css/chat.css index e63c64b..cb5e54e 100644 --- a/src/lembas/web/static/css/chat.css +++ b/src/lembas/web/static/css/chat.css @@ -412,6 +412,102 @@ } .chat-settings__params .field { margin-bottom: 0; } +/* --- Attachment chips (composer) ------------------------------------------ */ +.composer { position: relative; } + +.composer__attachments { + max-width: var(--thread-max-width); + margin: 0 auto var(--sp-2); + display: flex; + flex-wrap: wrap; + gap: var(--sp-2); +} +.composer__attachments:empty { display: none; } + +.chip { + display: flex; + align-items: center; + gap: var(--sp-2); + padding: var(--sp-2); + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--surface); + max-width: 20rem; + font-size: var(--text-sm); +} +.chip--error { border-color: var(--danger); background: var(--danger-soft); } +.chip--error .chip__icon { color: var(--danger); } + +.chip__thumb { + width: 2.25rem; + height: 2.25rem; + border-radius: var(--radius-sm); + object-fit: cover; + flex: none; +} +.chip__icon { color: var(--ink-muted); flex: none; display: flex; } +.chip__body { min-width: 0; flex: 1; display: flex; flex-direction: column; } +.chip__name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; } +.chip__meta { font-size: var(--text-xs); color: var(--ink-faint); } +.chip__warning { font-size: var(--text-xs); color: var(--danger); } + +.composer__attach { flex: none; align-self: flex-end; } + +/* --- Drag and drop -------------------------------------------------------- */ +.dropzone-overlay { + position: absolute; + inset: 0; + z-index: 5; + display: none; + flex-direction: column; + align-items: center; + justify-content: center; + gap: var(--sp-2); + border: 2px dashed var(--accent); + border-radius: var(--radius-lg); + background: color-mix(in srgb, var(--bg) 88%, var(--accent)); + color: var(--accent); + font-weight: 500; + /* The overlay must not eat the drop event it is advertising. */ + pointer-events: none; +} +.composer.is-dropping .dropzone-overlay { display: flex; } + +/* --- Attachments in the thread -------------------------------------------- */ +.attachments { + display: flex; + flex-wrap: wrap; + gap: var(--sp-2); + margin-bottom: var(--sp-2); +} +.attachments__image { + display: block; + border-radius: var(--radius); + overflow: hidden; + border: 1px solid var(--border); + line-height: 0; +} +.attachments__image img { + max-width: min(22rem, 100%); + max-height: 20rem; + width: auto; + height: auto; + object-fit: contain; +} +.attachments__doc { + display: flex; + align-items: flex-start; + gap: var(--sp-2); + padding: var(--sp-2) var(--sp-3); + border: 1px solid var(--border); + border-radius: var(--radius); + background: var(--surface); + font-size: var(--text-sm); + max-width: 100%; +} +.attachments__doc-body { display: flex; flex-direction: column; min-width: 0; } +.attachments__doc-body > a { overflow-wrap: anywhere; } + /* --- Model avatars -------------------------------------------------------- */ .model-avatar { width: 2rem; diff --git a/src/lembas/web/static/js/app.js b/src/lembas/web/static/js/app.js index 81ed765..ae2aa45 100644 --- a/src/lembas/web/static/js/app.js +++ b/src/lembas/web/static/js/app.js @@ -101,12 +101,100 @@ } } + /* --- Attachments ------------------------------------------------------- + Files are uploaded one at a time as soon as they are chosen, dropped or + pasted, rather than all at once when the message is sent. The chip (or the + rejection) then appears immediately, and a large file cannot make the send + button appear to hang. */ + function uploadFiles(fileList) { + var form = document.getElementById("upload-form"); + var target = document.getElementById("attachments"); + if (!form || !target || !fileList || !fileList.length) return; + + Array.prototype.forEach.call(fileList, function (file) { + var body = new FormData(); + body.append("file", file, file.name); + + fetch(form.getAttribute("hx-post"), { method: "POST", body: body }) + .then(function (response) { return response.text(); }) + .then(function (html) { + target.insertAdjacentHTML("beforeend", html); + // The chip's remove button is htmx-driven, so the new markup has to + // be announced or its attributes are inert. + if (window.htmx) window.htmx.process(target.lastElementChild); + }) + .catch(function () { + target.insertAdjacentHTML( + "beforeend", + '
' + + '' + + 'Upload failed.
' + ); + // Set as text, never as HTML: the filename comes from the user. + target.lastElementChild.querySelector(".chip__name").textContent = file.name; + }); + }); + } + + function setupDropzone() { + var zone = document.querySelector("[data-dropzone]"); + if (!zone) return; + + /* dragenter/dragleave fire for every child element the pointer crosses, so + a plain toggle flickers. Counting entries and exits is the standard fix. */ + var depth = 0; + + function hasFiles(event) { + return event.dataTransfer && Array.prototype.indexOf.call( + event.dataTransfer.types || [], "Files" + ) !== -1; + } + + zone.addEventListener("dragenter", function (event) { + if (!hasFiles(event)) return; + event.preventDefault(); + depth += 1; + zone.classList.add("is-dropping"); + }); + + zone.addEventListener("dragover", function (event) { + if (hasFiles(event)) event.preventDefault(); + }); + + zone.addEventListener("dragleave", function () { + depth = Math.max(0, depth - 1); + if (depth === 0) zone.classList.remove("is-dropping"); + }); + + zone.addEventListener("drop", function (event) { + if (!hasFiles(event)) return; + event.preventDefault(); + depth = 0; + zone.classList.remove("is-dropping"); + uploadFiles(event.dataTransfer.files); + }); + + /* Pasting a screenshot straight into the composer. Only files are taken; + pasted text must still behave as text. */ + document.addEventListener("paste", function (event) { + var composer = event.target.closest("[data-composer-input]"); + if (!composer || !event.clipboardData) return; + var files = Array.prototype.filter.call( + event.clipboardData.files || [], function (f) { return f && f.size; } + ); + if (!files.length) return; + event.preventDefault(); + uploadFiles(files); + }); + } + window.lembas = { applyTheme: applyTheme, toggleTheme: toggleTheme, copyText: copyText, scrollThread: scrollThread, - autosize: autosize + autosize: autosize, + uploadFiles: uploadFiles }; /* --- Wiring ------------------------------------------------------------ */ @@ -147,6 +235,7 @@ document.querySelectorAll("[data-autosize]").forEach(autosize); scrollThread(true); applyTheme(currentTheme()); + setupDropzone(); }); /* After any htmx swap: re-measure the composer and follow new content. */ diff --git a/src/lembas/web/templates/chat/_attachment_chip.html b/src/lembas/web/templates/chat/_attachment_chip.html new file mode 100644 index 0000000..1e6042c --- /dev/null +++ b/src/lembas/web/templates/chat/_attachment_chip.html @@ -0,0 +1,38 @@ +{% from "_macros.html" import icon %} +{# + A pending attachment in the composer, before the message is sent. + + Carries its own id in a hidden input so the composer form submits it with the + message; removing the chip removes the input, which is all the bookkeeping + the client needs. +#} +
+ + + {% if attachment.is_image %} + + {% else %} + + {{ icon("attach" if attachment.kind == "document" else "copy", "icon--sm") }} + + {% endif %} + + + {{ attachment.filename }} + + {{ attachment.human_size }} + {%- if attachment.pages %} · {{ attachment.pages }} page{{ '' if attachment.pages == 1 else 's' }}{% endif %} + {%- if attachment.width %} · {{ attachment.width }}×{{ attachment.height }}{% endif %} + {%- if attachment.truncated %} · truncated{% endif %} + + {% if attachment.extraction_error %} + {{ attachment.extraction_error }} + {% endif %} + + + +
diff --git a/src/lembas/web/templates/chat/_attachment_error.html b/src/lembas/web/templates/chat/_attachment_error.html new file mode 100644 index 0000000..d2d0466 --- /dev/null +++ b/src/lembas/web/templates/chat/_attachment_error.html @@ -0,0 +1,17 @@ +{% from "_macros.html" import icon %} +{# + A rejected upload. Rendered in place of a chip so the reason is visible in + the composer rather than only in the network tab. Dismissed by hand; it + carries no hidden input, so it cannot be submitted with the message. +#} +
+ {{ icon("warning", "icon--sm") }} + + {{ filename }} + {{ error }} + + +
diff --git a/src/lembas/web/templates/chat/_message.html b/src/lembas/web/templates/chat/_message.html index a9f8673..090fb46 100644 --- a/src/lembas/web/templates/chat/_message.html +++ b/src/lembas/web/templates/chat/_message.html @@ -38,6 +38,41 @@ {% endif %} + {% if message.attachments %} + {# Above the text, matching the order they were added and the order the + model receives them. #} +
+ {% for attachment in message.attachments %} + {% if attachment.is_image %} + + {{ attachment.filename }} + + {% else %} +
+ {{ icon("attach", "icon--sm") }} + + {{ attachment.filename }} + + {{ attachment.human_size }} + {%- if attachment.pages %} · {{ attachment.pages }} page{{ '' if attachment.pages == 1 else 's' }}{% endif %} + {%- if attachment.truncated %} · truncated{% endif %} + {%- if attachment.extracted_text %} + · view extracted text + {%- endif %} + + {% if attachment.extraction_error %} + {{ attachment.extraction_error }} + {% endif %} + +
+ {% endif %} + {% endfor %} +
+ {% endif %} + {% if streaming %} {# Reasoning arrives before the answer, so this block sits above it. It starts open (watching a model think is the point) and the :has() rule @@ -92,9 +127,11 @@ {% endif %} {% elif message.role == "assistant" %}
{{ body_html|safe }}
- {% else %} + {% elif message.content %}
{{ message.content }}
{% endif %} + {# An attachment-only turn has no text; rendering the bubble anyway would + leave an empty box under the file. #} {% if not streaming %}