File attachments: images for vision, PDFs and text into the prompt

Drag, paste or pick a file in the composer. Images go to vision models
as multimodal content parts; PDFs and text files have their content
extracted and placed in the prompt. Verified end to end against
gemma4-e4b-q8 on llama-swap: given a drawing and a text file, it named
the red square and blue circle and read the number out of the document.

Type is decided by inspecting the bytes, never the filename or the
browser's Content-Type -- a .png full of text is stored as text. Images
are downscaled to 1400px and re-encoded: a phone photo is several
megabytes of base64, which is slow and a large slice of the context
window. PDF text is extracted once, at upload, and stored; re-extracting
per request would let a reply change because a parser was upgraded.

Design points worth keeping:

- Images are only sent to models an administrator has marked `vision`.
  This is not graceful degradation -- most endpoints reject the entire
  request rather than ignoring an image part. A plain text turn stays a
  plain string for the same reason: the list form 400s on endpoints that
  do not implement it.
- Images reach the model as base64 data URIs, not links. A local
  endpoint has no route back to LLeMbas, and a hosted one has no
  credentials for it.
- Non-images are served Content-Disposition: attachment with nosniff, so
  an uploaded .html can never execute in this origin. Stored names are
  random; the uploader's name is a label and never a path.
- Uploads are unbound until the message is sent, which is what lets a
  file be removed beforehand. claim() only takes unclaimed rows owned by
  the sender, so a forged id cannot pull in someone else's file.
  Abandoned uploads are swept at startup.
- A scanned PDF says so rather than silently contributing nothing, and
  truncation is declared to the model in the document tag so it can
  admit it did not see page 400.
- "Here, look at this" with no words is a legitimate turn, so a message
  is only empty when it carries neither text nor files.

Also fixes auto-titling, which read message["content"] as a string and
would have broken on the first multimodal turn.

186 tests, ruff clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
2026-07-21 12:19:59 +02:00
parent 1d3f6c450b
commit d90195015c
18 changed files with 1571 additions and 14 deletions
+5 -2
View File
@@ -36,6 +36,9 @@ runtime. Clone it, `pip install -e .`, run it.
- **Reasoning display** — thinking from reasoning models streams into its own - **Reasoning display** — thinking from reasoning models streams into its own
collapsible block, labelled with how long it took, and is never replayed as collapsible block, labelled with how long it took, and is never replayed as
context context
- **Attachments** — drag, paste or pick images, PDFs and text files. Images are
downscaled and sent to vision models; PDF and text content is extracted and
put in the prompt
- **Folders** — arbitrarily nested, delete a folder without losing the chats - **Folders** — arbitrarily nested, delete a folder without losing the chats
inside it inside it
- **OpenAI connections** — point at OpenAI, LM Studio, vLLM, llama.cpp, - **OpenAI connections** — point at OpenAI, LM Studio, vLLM, llama.cpp,
@@ -53,8 +56,8 @@ runtime. Clone it, `pip install -e .`, run it.
**Planned** **Planned**
File upload, vision and PDFs · built-in tools with admin settings · custom tools Built-in tools with admin settings · custom tools and MCP servers · agentic
and MCP servers · agentic execution (local and over SSH) · image generation. execution (local and over SSH) · image generation · OCR for scanned PDFs.
## Quick start ## Quick start
+2
View File
@@ -33,6 +33,8 @@ dependencies = [
"linkify-it-py>=2.0", # bare URLs in model output become links "linkify-it-py>=2.0", # bare URLs in model output become links
"pygments>=2.18", "pygments>=2.18",
"nh3>=0.2.18", "nh3>=0.2.18",
"pypdf>=5.1", # PDF text extraction for attachments
"pillow>=11.0", # image validation and downscaling for vision
"typer>=0.12", "typer>=0.12",
] ]
+29 -3
View File
@@ -16,6 +16,7 @@ from lembas.db.models import ROLE_ASSISTANT, ROLE_USER, Chat, Message, User
from lembas.db.session import session_scope from lembas.db.session import session_scope
from lembas.security import permissions from lembas.security import permissions
from lembas.services import chat as chat_service from lembas.services import chat as chat_service
from lembas.services import files as files_service
from lembas.services import sse from lembas.services import sse
from lembas.services.llm.openai_client import ( from lembas.services.llm.openai_client import (
LLMError, LLMError,
@@ -66,7 +67,8 @@ async def post_message(
db: Db, db: Db,
user: RequiredUser, user: RequiredUser,
chat_id: str, chat_id: str,
content: str = Form(...), content: str = Form(""),
file_ids: list[str] = Form(default=[]),
) -> Response: ) -> Response:
"""Persist the user's turn and hand back the pair of bubbles. """Persist the user's turn and hand back the pair of bubbles.
@@ -77,10 +79,15 @@ async def post_message(
chat = _owned_chat(db, chat_id, user.id) chat = _owned_chat(db, chat_id, user.id)
content = content.strip() content = content.strip()
if not content: # "Here, look at this" with no words is a legitimate turn, so an empty
# message is only empty when it carries nothing at all.
if not content and not file_ids:
return Response(status_code=status.HTTP_204_NO_CONTENT) return Response(status_code=status.HTTP_204_NO_CONTENT)
user_message = chat_service.create_message(db, chat, ROLE_USER, content) user_message = chat_service.create_message(db, chat, ROLE_USER, content)
if file_ids:
files_service.claim(db, ids=file_ids, user_id=user.id, message_id=user_message.id)
db.refresh(user_message)
assistant_message = chat_service.create_message( assistant_message = chat_service.create_message(
db, chat, ROLE_ASSISTANT, "", complete_=False, model_id=chat.model_id db, chat, ROLE_ASSISTANT, "", complete_=False, model_id=chat.model_id
) )
@@ -130,6 +137,19 @@ async def stream_message(
) )
def _plain_text(content: str | list) -> str:
"""The text of a message payload, whether it is a string or content parts."""
if isinstance(content, str):
return content
if isinstance(content, list):
return " ".join(
part.get("text", "")
for part in content
if isinstance(part, dict) and part.get("type") == "text"
).strip()
return ""
async def _generate(chat_id: str, message_id: str) -> AsyncIterator[str]: async def _generate(chat_id: str, message_id: str) -> AsyncIterator[str]:
"""Drive one completion and frame it as SSE. """Drive one completion and frame it as SSE.
@@ -159,8 +179,14 @@ async def _generate(chat_id: str, message_id: str) -> AsyncIterator[str]:
try: try:
endpoint, model_id = chat_service.resolve_endpoint(db, chat) endpoint, model_id = chat_service.resolve_endpoint(db, chat)
payload = chat_service.build_request(db, chat, upto=message) payload = chat_service.build_request(db, chat, upto=message)
# A multimodal turn's content is a list of parts, not a string, so
# the text has to be picked out before it can title a chat.
first_user_text = next( first_user_text = next(
(m["content"] for m in reversed(payload["messages"]) if m["role"] == ROLE_USER), (
_plain_text(m["content"])
for m in reversed(payload["messages"])
if m["role"] == ROLE_USER
),
"", "",
) )
+117
View File
@@ -0,0 +1,117 @@
"""Uploading, serving and removing chat attachments."""
from __future__ import annotations
import logging
from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status
from fastapi.responses import FileResponse
from lembas.api.deps import Db, RequiredUser, require_permission
from lembas.db.models import Attachment
from lembas.services import files as files_service
from lembas.web.templating import templates
log = logging.getLogger(__name__)
router = APIRouter(prefix="/api/files", tags=["files"])
def _owned(db: Db, attachment_id: str, user_id: str) -> Attachment:
attachment = db.get(Attachment, attachment_id)
if attachment is None or attachment.user_id != user_id:
raise HTTPException(status.HTTP_404_NOT_FOUND, "That file no longer exists.")
return attachment
@router.post("", dependencies=[Depends(require_permission("files.upload"))])
async def upload(
request: Request,
db: Db,
user: RequiredUser,
file: UploadFile = File(...),
chat_id: str = "",
) -> Response:
"""Accept one file and return the chip that represents it in the composer.
The attachment is stored immediately but left unbound: it only joins a
message when that message is sent. That is what lets a file be removed
before sending, and what the orphan sweep later cleans up.
"""
payload = await file.read()
try:
attachment = files_service.store(
db,
user_id=user.id,
chat_id=chat_id or None,
payload=payload,
filename=file.filename or "file",
)
except files_service.FileError as exc:
# 200 with an error chip rather than a 4xx: htmx swaps the response
# body either way, and an error the user can read beats a silent
# failure in the console.
return templates.TemplateResponse(
request,
"chat/_attachment_error.html",
{"request": request, "filename": file.filename or "file", "error": str(exc)},
)
return templates.TemplateResponse(
request,
"chat/_attachment_chip.html",
{"request": request, "attachment": attachment},
)
@router.delete("/{attachment_id}")
async def remove(db: Db, user: RequiredUser, attachment_id: str) -> Response:
"""Detach a file before it has been sent."""
attachment = _owned(db, attachment_id, user.id)
if attachment.message_id is not None:
# Deleting it now would rewrite a conversation that has already been
# sent to a model and read by the user.
raise HTTPException(
status.HTTP_409_CONFLICT, "That file is part of a sent message."
)
files_service.delete(db, attachment)
return Response(status_code=status.HTTP_200_OK)
@router.get("/{attachment_id}/content")
async def content(db: Db, user: RequiredUser, attachment_id: str) -> Response:
"""Serve an attachment back to its owner."""
attachment = _owned(db, attachment_id, user.id)
path = files_service.stored_path(attachment.stored_name)
if path is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, "That file is no longer on disk.")
# inline for images so they render in the thread; attachment for everything
# else so a text/html upload can never be executed in this origin.
disposition = "inline" if attachment.is_image else "attachment"
return FileResponse(
path,
media_type=attachment.media_type if attachment.is_image else "application/octet-stream",
headers={
"Content-Disposition": f'{disposition}; filename="{attachment.filename}"',
"Cache-Control": "private, max-age=604800",
# Belt and braces: even for images, never let a browser sniff its
# way to treating the bytes as something executable.
"X-Content-Type-Options": "nosniff",
},
)
@router.get("/{attachment_id}/text")
async def extracted_text(db: Db, user: RequiredUser, attachment_id: str) -> Response:
"""The text a document contributed to the prompt.
Worth being able to see: a PDF that extracted badly explains a strange
reply, and there is otherwise no way to tell what the model was given.
"""
attachment = _owned(db, attachment_id, user.id)
return Response(
attachment.extracted_text or attachment.extraction_error,
media_type="text/plain; charset=utf-8",
)
+10
View File
@@ -5,6 +5,12 @@ what ``init_db()`` relies on to create the schema at startup. Any new model
module must be imported here or its table will silently never be created. module must be imported here or its table will silently never be created.
""" """
from lembas.db.models.attachment import (
KIND_DOCUMENT,
KIND_IMAGE,
KIND_TEXT,
Attachment,
)
from lembas.db.models.chat import ( from lembas.db.models.chat import (
ROLE_ASSISTANT, ROLE_ASSISTANT,
ROLE_SYSTEM, ROLE_SYSTEM,
@@ -26,6 +32,10 @@ from lembas.db.models.user import (
) )
__all__ = [ __all__ = [
"Attachment",
"KIND_DOCUMENT",
"KIND_IMAGE",
"KIND_TEXT",
"ROLE_ADMIN", "ROLE_ADMIN",
"ROLE_ASSISTANT", "ROLE_ASSISTANT",
"ROLE_PENDING", "ROLE_PENDING",
+74
View File
@@ -0,0 +1,74 @@
"""Files attached to chat messages."""
from __future__ import annotations
from sqlalchemy import Boolean, ForeignKey, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column, relationship
from lembas.db.base import Base, Timestamps, UUIDPrimaryKey
# What the file is for, decided at upload time. Drives both how it is rendered
# and how it reaches the model: images become multimodal parts, everything else
# becomes text in the prompt.
KIND_IMAGE = "image"
KIND_DOCUMENT = "document" # PDF: text is extracted
KIND_TEXT = "text" # plain text, markdown, csv, source code
class Attachment(UUIDPrimaryKey, Timestamps, Base):
__tablename__ = "attachments"
user_id: Mapped[str] = mapped_column(
String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
)
chat_id: Mapped[str | None] = mapped_column(
String(32), ForeignKey("chats.id", ondelete="CASCADE"), index=True
)
# Null while the file is uploaded but the message has not been sent yet.
# Those orphans are swept periodically -- see services.files.sweep_orphans.
message_id: Mapped[str | None] = mapped_column(
String(32), ForeignKey("messages.id", ondelete="CASCADE"), index=True
)
# What the uploader called it. Display only, never used as a path.
filename: Mapped[str] = mapped_column(String(300), nullable=False)
# Random name on disk. See services.files for why the two are separate.
stored_name: Mapped[str] = mapped_column(String(120), nullable=False)
media_type: Mapped[str] = mapped_column(String(100), default="")
size_bytes: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
kind: Mapped[str] = mapped_column(String(16), default=KIND_DOCUMENT, nullable=False)
# Images only, after downscaling.
width: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
height: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
# Documents and text: the content that actually reaches the model. Held in
# the database rather than re-extracted per request -- extraction is slow,
# and a reply must not silently change because a PDF parser was upgraded.
extracted_text: Mapped[str] = mapped_column(Text, default="")
pages: Mapped[int] = mapped_column(Integer, default=0, nullable=False)
truncated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
# Non-empty when the file was stored but its text could not be read, e.g. a
# scanned PDF with no text layer. Shown next to the attachment so the user
# is not left wondering why the model ignored it.
extraction_error: Mapped[str] = mapped_column(Text, default="")
message: Mapped[Message] = relationship(back_populates="attachments") # noqa: F821
@property
def is_image(self) -> bool:
return self.kind == KIND_IMAGE
@property
def human_size(self) -> str:
size = float(self.size_bytes)
for unit in ("B", "KB", "MB"):
if size < 1024 or unit == "MB":
return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}"
size /= 1024
return f"{size:.1f} MB"
def __repr__(self) -> str:
return f"<Attachment {self.filename} {self.kind}>"
+13
View File
@@ -120,6 +120,19 @@ class Message(UUIDPrimaryKey, Timestamps, Base):
complete: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) complete: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False)
chat: Mapped[Chat] = relationship(back_populates="messages") chat: Mapped[Chat] = relationship(back_populates="messages")
attachments: Mapped[list[Attachment]] = relationship( # noqa: F821
back_populates="message",
cascade="all, delete-orphan",
order_by="Attachment.created_at",
)
@property
def images(self) -> list:
return [a for a in self.attachments if a.is_image]
@property
def documents(self) -> list:
return [a for a in self.attachments if not a.is_image]
def __repr__(self) -> str: def __repr__(self) -> str:
return f"<Message {self.role} {self.content[:40]!r}>" return f"<Message {self.role} {self.content[:40]!r}>"
+13
View File
@@ -18,6 +18,7 @@ from lembas.api import (
admin_users, admin_users,
auth, auth,
chats, chats,
files,
folders, folders,
pages, pages,
preferences, preferences,
@@ -52,6 +53,17 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
' python -c "import secrets; print(secrets.token_urlsafe(48))"' ' python -c "import secrets; print(secrets.token_urlsafe(48))"'
) )
# Files chosen in a composer that was never sent would otherwise sit on
# disk forever. Cheap, and startup is the natural moment for it.
try:
from lembas.db.session import session_scope
from lembas.services.files import sweep_orphans
with session_scope() as db:
sweep_orphans(db)
except Exception: # noqa: BLE001 - housekeeping must never block startup
log.exception("orphaned upload sweep failed")
log.info("LLeMbas %s starting on http://%s:%s", __version__, settings.host, settings.port) log.info("LLeMbas %s starting on http://%s:%s", __version__, settings.host, settings.port)
log.info("data directory: %s", settings.data_dir.resolve()) log.info("data directory: %s", settings.data_dir.resolve())
yield yield
@@ -74,6 +86,7 @@ def create_app() -> FastAPI:
app.include_router(auth.router) app.include_router(auth.router)
app.include_router(preferences.router) app.include_router(preferences.router)
app.include_router(chats.router) app.include_router(chats.router)
app.include_router(files.router)
app.include_router(folders.router) app.include_router(folders.router)
app.include_router(admin.router) app.include_router(admin.router)
app.include_router(admin_users.router) app.include_router(admin_users.router)
+8
View File
@@ -70,6 +70,14 @@ PERMISSION_DEFS: tuple[PermissionDef, ...] = (
True, True,
"Workspace", "Workspace",
), ),
PermissionDef(
"files.upload",
"Attach files",
"Attach images, PDFs and text files to a message. Images only reach "
"models marked as having vision.",
True,
"Workspace",
),
) )
PERMISSION_KEYS = tuple(d.key for d in PERMISSION_DEFS) PERMISSION_KEYS = tuple(d.key for d in PERMISSION_DEFS)
+80 -6
View File
@@ -17,6 +17,7 @@ from lembas.db.models import (
Message, Message,
Model, Model,
) )
from lembas.services import files as files_service
from lembas.services.llm.openai_client import Endpoint, LLMError, complete from lembas.services.llm.openai_client import Endpoint, LLMError, complete
log = logging.getLogger(__name__) log = logging.getLogger(__name__)
@@ -71,7 +72,66 @@ def resolve_endpoint(db: DBSession, chat: Chat) -> tuple[Endpoint, str]:
return Endpoint.from_connection(connection), chat.model_id return Endpoint.from_connection(connection), chat.model_id
def build_messages(db: DBSession, chat: Chat, *, upto: Message | None = None) -> list[dict]: def document_context(message: Message) -> str:
"""Extracted text from a message's non-image attachments.
Wrapped in named tags so the model can tell one document from another, and
tell all of them from what the user actually typed. Truncation is stated
inline rather than silently, so a model asked about page 400 of a 300-page
extract can say it did not see it.
"""
blocks: list[str] = []
for attachment in message.documents:
if not attachment.extracted_text.strip():
continue
note = " (truncated)" if attachment.truncated else ""
blocks.append(
f'<document name="{attachment.filename}"{note}>\n'
f"{attachment.extracted_text.strip()}\n"
f"</document>"
)
return "\n\n".join(blocks)
def message_payload(message: Message, *, vision: bool) -> dict[str, Any]:
"""One history entry in the shape the endpoint expects.
Plain text stays a plain string: sending the multimodal list form to an
endpoint that does not implement it is a reliable way to get a 400, and
most local runners do not.
"""
text = message.content.strip()
documents = document_context(message)
if documents:
# Documents lead so the question that follows has its material already
# in view, which is how these models are trained to read a prompt.
text = f"{documents}\n\n{text}" if text else documents
images = message.images if vision else []
if not images:
return {"role": message.role, "content": text}
parts: list[dict[str, Any]] = []
if text:
parts.append({"type": "text", "text": text})
for attachment in images:
uri = files_service.data_uri(attachment)
if uri is None:
# The row survived but the file did not. Better to say so than to
# send a turn that silently lost its picture.
log.warning("attachment %s has no file on disk", attachment.id)
continue
parts.append({"type": "image_url", "image_url": {"url": uri}})
if not parts:
return {"role": message.role, "content": text}
return {"role": message.role, "content": parts}
def build_messages(
db: DBSession, chat: Chat, *, upto: Message | None = None, vision: bool = False
) -> list[dict]:
"""Assemble the message list to send upstream. """Assemble the message list to send upstream.
`upto` excludes the placeholder assistant row being generated into, and `upto` excludes the placeholder assistant row being generated into, and
@@ -88,24 +148,38 @@ def build_messages(db: DBSession, chat: Chat, *, upto: Message | None = None) ->
for message in history: for message in history:
if upto is not None and message.id == upto.id: if upto is not None and message.id == upto.id:
break break
# Skip turns that failed or produced nothing: sending an empty # Skip turns that failed or produced nothing -- but a message carrying
# assistant message upsets several providers. # only an attachment has no text and must still be sent.
if message.error or not message.content.strip(): if message.error:
continue continue
payload.append({"role": message.role, "content": message.content}) if not message.content.strip() and not message.attachments:
continue
payload.append(message_payload(message, vision=vision))
return payload return payload
def model_supports(db: DBSession, chat: Chat, capability: str) -> bool:
"""Whether the chat's current model is marked as having a capability."""
model = db.scalar(
select(Model).where(Model.model_id == chat.model_id).order_by(Model.position)
)
return bool(model and (model.capabilities_json or {}).get(capability))
def build_request(db: DBSession, chat: Chat, *, upto: Message | None = None) -> dict[str, Any]: def build_request(db: DBSession, chat: Chat, *, upto: Message | None = None) -> dict[str, Any]:
params = { params = {
key: value key: value
for key, value in (chat.params_json or {}).items() for key, value in (chat.params_json or {}).items()
if key in FORWARDED_PARAMS and value not in (None, "") if key in FORWARDED_PARAMS and value not in (None, "")
} }
# Images are only sent to a model an administrator has marked as having
# vision. Sending them to one that has not is not a graceful degradation:
# most endpoints reject the whole request.
vision = model_supports(db, chat, "vision")
return { return {
"model": chat.model_id, "model": chat.model_id,
"messages": build_messages(db, chat, upto=upto), "messages": build_messages(db, chat, upto=upto, vision=vision),
**params, **params,
} }
+400
View File
@@ -0,0 +1,400 @@
"""Storing and reading uploaded attachments.
Three kinds of file, each handled differently on the way to the model:
* **Images** are downscaled and re-encoded, then sent as multimodal content
parts. Downscaling is not cosmetic -- a phone photo is several megabytes of
base64, which is both slow and a large slice of the context window.
* **PDFs** have their text extracted once, at upload. Extraction is slow and a
reply must not silently change because a parser was upgraded later.
* **Plain text** (including source code and CSV) is decoded and stored as-is.
Everything an uploader supplies is treated as hostile: the type is decided by
inspecting the bytes rather than trusting the browser, the name on disk is
random, and both image dimensions and PDF page counts are capped so a small
file cannot expand into an enormous amount of work.
"""
from __future__ import annotations
import io
import logging
import secrets
from dataclasses import dataclass
from datetime import UTC, datetime, timedelta
from pathlib import Path
from PIL import Image, UnidentifiedImageError
from sqlalchemy import select
from sqlalchemy.orm import Session as DBSession
from lembas.config import settings
from lembas.db.models import KIND_DOCUMENT, KIND_IMAGE, KIND_TEXT, Attachment
log = logging.getLogger(__name__)
# --- Limits ------------------------------------------------------------------
MAX_UPLOAD_BYTES = 20 * 1024 * 1024
# Longest edge after downscaling. Large enough for a model to read a screenshot
# or a page of text, small enough that the base64 stays reasonable.
MAX_IMAGE_EDGE = 1400
JPEG_QUALITY = 85
# Pillow's own guard against decompression bombs: a 60,000x60,000 PNG is a few
# KB on disk and hundreds of GB decoded.
Image.MAX_IMAGE_PIXELS = 64_000_000
MAX_PDF_PAGES = 300
# Characters of extracted text kept per document. Roughly 30k tokens, which is
# already a large slice of most context windows; more is rarely useful and
# frequently breaks the request outright.
MAX_EXTRACTED_CHARS = 120_000
# Orphans are files uploaded into a composer that was never sent.
ORPHAN_AGE = timedelta(hours=24)
IMAGE_TYPES: dict[bytes, tuple[str, str]] = {
b"\x89PNG\r\n\x1a\n": ("image/png", ".png"),
b"\xff\xd8\xff": ("image/jpeg", ".jpg"),
b"GIF87a": ("image/gif", ".gif"),
b"GIF89a": ("image/gif", ".gif"),
}
# Extensions treated as text when the bytes decode cleanly as UTF-8. The list
# exists only to pick a sensible media type; decodability is what actually
# decides, so an unlisted extension still works.
TEXT_EXTENSIONS = {
".txt": "text/plain", ".md": "text/markdown", ".markdown": "text/markdown",
".csv": "text/csv", ".tsv": "text/tab-separated-values",
".json": "application/json", ".yaml": "text/yaml", ".yml": "text/yaml",
".toml": "text/toml", ".ini": "text/plain", ".cfg": "text/plain",
".xml": "text/xml", ".html": "text/plain", ".css": "text/plain",
".py": "text/x-python", ".js": "text/javascript", ".ts": "text/typescript",
".rs": "text/x-rust", ".go": "text/x-go", ".c": "text/x-c", ".h": "text/x-c",
".cpp": "text/x-c++", ".java": "text/x-java", ".rb": "text/x-ruby",
".sh": "text/x-shellscript", ".sql": "text/x-sql", ".log": "text/plain",
}
class FileError(Exception):
"""A rejected upload, with a message fit to show the user."""
@dataclass
class Prepared:
"""The result of inspecting and processing an upload, before it is stored."""
payload: bytes
kind: str
media_type: str
extension: str
width: int = 0
height: int = 0
extracted_text: str = ""
pages: int = 0
truncated: bool = False
extraction_error: str = ""
# --- Storage -----------------------------------------------------------------
def attachments_dir() -> Path:
path = settings.uploads_dir / "attachments"
path.mkdir(parents=True, exist_ok=True)
return path
def stored_path(stored_name: str) -> Path | None:
"""Resolve a stored name to a path, refusing anything outside the directory."""
if not stored_name or "/" in stored_name or "\\" in stored_name or stored_name.startswith("."):
return None
base = attachments_dir().resolve()
path = (base / stored_name).resolve()
try:
path.relative_to(base)
except ValueError:
return None
return path if path.is_file() else None
# --- Type detection ----------------------------------------------------------
def _detect_image(payload: bytes) -> tuple[str, str] | None:
for signature, (media_type, extension) in IMAGE_TYPES.items():
if payload.startswith(signature):
return media_type, extension
if payload[:4] == b"RIFF" and payload[8:12] == b"WEBP":
return "image/webp", ".webp"
return None
def _looks_like_pdf(payload: bytes) -> bool:
# The header is allowed a little leading junk by the spec, and real files
# in the wild use it.
return b"%PDF-" in payload[:1024]
# --- Processing --------------------------------------------------------------
def _process_image(payload: bytes) -> Prepared:
try:
with Image.open(io.BytesIO(payload)) as image:
image.load()
has_alpha = image.mode in ("RGBA", "LA", "P") and "transparency" in image.info
# Animation is lost on re-encode; keeping only the first frame is
# honest and is what a model would see anyway.
frame = image.convert("RGBA" if has_alpha else "RGB")
width, height = frame.size
longest = max(width, height)
if longest > MAX_IMAGE_EDGE:
scale = MAX_IMAGE_EDGE / longest
frame = frame.resize(
(max(1, int(width * scale)), max(1, int(height * scale))),
Image.LANCZOS,
)
buffer = io.BytesIO()
if has_alpha:
frame.save(buffer, format="PNG", optimize=True)
media_type, extension = "image/png", ".png"
else:
frame.save(buffer, format="JPEG", quality=JPEG_QUALITY, optimize=True)
media_type, extension = "image/jpeg", ".jpg"
return Prepared(
payload=buffer.getvalue(),
kind=KIND_IMAGE,
media_type=media_type,
extension=extension,
width=frame.width,
height=frame.height,
)
except Image.DecompressionBombError as exc:
raise FileError("That image's dimensions are implausibly large.") from exc
except (UnidentifiedImageError, OSError, ValueError) as exc:
raise FileError("That image could not be read. Is it corrupt?") from exc
def _process_pdf(payload: bytes) -> Prepared:
from pypdf import PdfReader
from pypdf.errors import PdfReadError
prepared = Prepared(
payload=payload, kind=KIND_DOCUMENT, media_type="application/pdf", extension=".pdf"
)
try:
reader = PdfReader(io.BytesIO(payload))
if reader.is_encrypted:
# An empty password unlocks a surprising number of "encrypted" PDFs.
try:
reader.decrypt("")
except Exception: # noqa: BLE001 - any failure means the same thing
prepared.extraction_error = (
"This PDF is password-protected, so its text could not be read."
)
return prepared
prepared.pages = len(reader.pages)
chunks: list[str] = []
total = 0
for index, page in enumerate(reader.pages[:MAX_PDF_PAGES]):
try:
text = page.extract_text() or ""
except Exception as exc: # noqa: BLE001 - one bad page is not fatal
log.debug("page %d of a PDF failed to extract: %s", index, exc)
continue
if not text.strip():
continue
chunks.append(f"[page {index + 1}]\n{text.strip()}")
total += len(text)
if total >= MAX_EXTRACTED_CHARS:
prepared.truncated = True
break
if prepared.pages > MAX_PDF_PAGES:
prepared.truncated = True
prepared.extracted_text = "\n\n".join(chunks)[:MAX_EXTRACTED_CHARS]
if not prepared.extracted_text.strip():
# Almost always a scan. Saying so beats the model silently ignoring
# a document the user believes it can read.
prepared.extraction_error = (
"No text could be extracted. This looks like a scanned PDF; "
"LLeMbas does not do OCR yet."
)
except PdfReadError as exc:
prepared.extraction_error = "This file is not a readable PDF."
log.info("unreadable PDF: %s", exc)
except Exception as exc: # noqa: BLE001 - never let a bad file 500 the upload
prepared.extraction_error = "This PDF could not be read."
log.warning("unexpected PDF failure: %s", exc)
return prepared
def _process_text(payload: bytes, filename: str) -> Prepared:
for encoding in ("utf-8", "utf-16", "latin-1"):
try:
text = payload.decode(encoding)
break
except (UnicodeDecodeError, LookupError):
continue
else:
raise FileError("That file is not text, and is not a format LLeMbas can read.")
# Null bytes mean this decoded by luck (latin-1 decodes any byte) and is
# really a binary file.
if "\x00" in text[:4096]:
raise FileError("That file is not text, and is not a format LLeMbas can read.")
truncated = len(text) > MAX_EXTRACTED_CHARS
extension = Path(filename).suffix.lower()
return Prepared(
payload=payload,
kind=KIND_TEXT,
media_type=TEXT_EXTENSIONS.get(extension, "text/plain"),
extension=extension if extension in TEXT_EXTENSIONS else ".txt",
extracted_text=text[:MAX_EXTRACTED_CHARS],
truncated=truncated,
)
def prepare(payload: bytes, filename: str) -> Prepared:
"""Inspect an upload, decide what it is, and process it accordingly."""
if not payload:
raise FileError("That file is empty.")
if len(payload) > MAX_UPLOAD_BYTES:
raise FileError(f"Files must be under {MAX_UPLOAD_BYTES // (1024 * 1024)} MB.")
if _detect_image(payload) is not None:
return _process_image(payload)
if _looks_like_pdf(payload):
return _process_pdf(payload)
return _process_text(payload, filename)
# --- Public API --------------------------------------------------------------
def safe_display_name(filename: str) -> str:
"""A filename fit to show. Never used as a path; the stored name is random."""
cleaned = Path(filename or "file").name.strip() or "file"
return cleaned[:300]
def store(
db: DBSession,
*,
user_id: str,
chat_id: str | None,
payload: bytes,
filename: str,
) -> Attachment:
"""Process and persist an upload. Raises FileError if it is unusable."""
prepared = prepare(payload, filename)
stored_name = f"{secrets.token_hex(16)}{prepared.extension}"
(attachments_dir() / stored_name).write_bytes(prepared.payload)
attachment = Attachment(
user_id=user_id,
chat_id=chat_id,
filename=safe_display_name(filename),
stored_name=stored_name,
media_type=prepared.media_type,
size_bytes=len(prepared.payload),
kind=prepared.kind,
width=prepared.width,
height=prepared.height,
extracted_text=prepared.extracted_text,
pages=prepared.pages,
truncated=prepared.truncated,
extraction_error=prepared.extraction_error,
)
db.add(attachment)
db.commit()
log.info(
"stored %s (%s, %d bytes) for user %s",
attachment.filename,
attachment.kind,
attachment.size_bytes,
user_id,
)
return attachment
def delete(db: DBSession, attachment: Attachment) -> None:
path = stored_path(attachment.stored_name)
if path is not None:
path.unlink(missing_ok=True)
db.delete(attachment)
db.commit()
def claim(db: DBSession, *, ids: list[str], user_id: str, message_id: str) -> list[Attachment]:
"""Bind pending uploads to the message that was just sent.
Only unclaimed attachments belonging to this user are taken, so a stray or
forged id cannot pull someone else's file into a conversation.
"""
if not ids:
return []
pending = list(
db.scalars(
select(Attachment).where(
Attachment.id.in_(ids),
Attachment.user_id == user_id,
Attachment.message_id.is_(None),
)
)
)
for attachment in pending:
attachment.message_id = message_id
db.commit()
return pending
def sweep_orphans(db: DBSession, older_than: timedelta = ORPHAN_AGE) -> int:
"""Delete uploads that were never attached to a message.
A file picked in the composer and then abandoned would otherwise sit on
disk forever.
"""
cutoff = datetime.now(UTC) - older_than
orphans = list(db.scalars(select(Attachment).where(Attachment.message_id.is_(None))))
removed = 0
for attachment in orphans:
created = attachment.created_at
if created.tzinfo is None:
created = created.replace(tzinfo=UTC)
if created >= cutoff:
continue
path = stored_path(attachment.stored_name)
if path is not None:
path.unlink(missing_ok=True)
db.delete(attachment)
removed += 1
if removed:
db.commit()
log.info("swept %d orphaned upload(s)", removed)
return removed
def data_uri(attachment: Attachment) -> str | None:
"""Base64 data URI for an image, as sent to a vision model.
A data URI rather than a link back to this server: a local endpoint has no
route to LLeMbas, and a hosted one has no credentials for it.
"""
import base64
path = stored_path(attachment.stored_name)
if path is None:
return None
encoded = base64.b64encode(path.read_bytes()).decode("ascii")
return f"data:{attachment.media_type};base64,{encoded}"
+96
View File
@@ -412,6 +412,102 @@
} }
.chat-settings__params .field { margin-bottom: 0; } .chat-settings__params .field { margin-bottom: 0; }
/* --- Attachment chips (composer) ------------------------------------------ */
.composer { position: relative; }
.composer__attachments {
max-width: var(--thread-max-width);
margin: 0 auto var(--sp-2);
display: flex;
flex-wrap: wrap;
gap: var(--sp-2);
}
.composer__attachments:empty { display: none; }
.chip {
display: flex;
align-items: center;
gap: var(--sp-2);
padding: var(--sp-2);
border: 1px solid var(--border);
border-radius: var(--radius);
background: var(--surface);
max-width: 20rem;
font-size: var(--text-sm);
}
.chip--error { border-color: var(--danger); background: var(--danger-soft); }
.chip--error .chip__icon { color: var(--danger); }
.chip__thumb {
width: 2.25rem;
height: 2.25rem;
border-radius: var(--radius-sm);
object-fit: cover;
flex: none;
}
.chip__icon { color: var(--ink-muted); flex: none; display: flex; }
.chip__body { min-width: 0; flex: 1; display: flex; flex-direction: column; }
.chip__name { overflow: hidden; text-overflow: ellipsis; white-space: nowrap; }
.chip__meta { font-size: var(--text-xs); color: var(--ink-faint); }
.chip__warning { font-size: var(--text-xs); color: var(--danger); }
.composer__attach { flex: none; align-self: flex-end; }
/* --- Drag and drop -------------------------------------------------------- */
.dropzone-overlay {
position: absolute;
inset: 0;
z-index: 5;
display: none;
flex-direction: column;
align-items: center;
justify-content: center;
gap: var(--sp-2);
border: 2px dashed var(--accent);
border-radius: var(--radius-lg);
background: color-mix(in srgb, var(--bg) 88%, var(--accent));
color: var(--accent);
font-weight: 500;
/* The overlay must not eat the drop event it is advertising. */
pointer-events: none;
}
.composer.is-dropping .dropzone-overlay { display: flex; }
/* --- Attachments in the thread -------------------------------------------- */
.attachments {
display: flex;
flex-wrap: wrap;
gap: var(--sp-2);
margin-bottom: var(--sp-2);
}
.attachments__image {
display: block;
border-radius: var(--radius);
overflow: hidden;
border: 1px solid var(--border);
line-height: 0;
}
.attachments__image img {
max-width: min(22rem, 100%);
max-height: 20rem;
width: auto;
height: auto;
object-fit: contain;
}
.attachments__doc {
display: flex;
align-items: flex-start;
gap: var(--sp-2);
padding: var(--sp-2) var(--sp-3);
border: 1px solid var(--border);
border-radius: var(--radius);
background: var(--surface);
font-size: var(--text-sm);
max-width: 100%;
}
.attachments__doc-body { display: flex; flex-direction: column; min-width: 0; }
.attachments__doc-body > a { overflow-wrap: anywhere; }
/* --- Model avatars -------------------------------------------------------- */ /* --- Model avatars -------------------------------------------------------- */
.model-avatar { .model-avatar {
width: 2rem; width: 2rem;
+90 -1
View File
@@ -101,12 +101,100 @@
} }
} }
/* --- Attachments -------------------------------------------------------
Files are uploaded one at a time as soon as they are chosen, dropped or
pasted, rather than all at once when the message is sent. The chip (or the
rejection) then appears immediately, and a large file cannot make the send
button appear to hang. */
function uploadFiles(fileList) {
var form = document.getElementById("upload-form");
var target = document.getElementById("attachments");
if (!form || !target || !fileList || !fileList.length) return;
Array.prototype.forEach.call(fileList, function (file) {
var body = new FormData();
body.append("file", file, file.name);
fetch(form.getAttribute("hx-post"), { method: "POST", body: body })
.then(function (response) { return response.text(); })
.then(function (html) {
target.insertAdjacentHTML("beforeend", html);
// The chip's remove button is htmx-driven, so the new markup has to
// be announced or its attributes are inert.
if (window.htmx) window.htmx.process(target.lastElementChild);
})
.catch(function () {
target.insertAdjacentHTML(
"beforeend",
'<div class="chip chip--error"><span class="chip__body">' +
'<span class="chip__name"></span>' +
'<span class="chip__warning">Upload failed.</span></span></div>'
);
// Set as text, never as HTML: the filename comes from the user.
target.lastElementChild.querySelector(".chip__name").textContent = file.name;
});
});
}
function setupDropzone() {
var zone = document.querySelector("[data-dropzone]");
if (!zone) return;
/* dragenter/dragleave fire for every child element the pointer crosses, so
a plain toggle flickers. Counting entries and exits is the standard fix. */
var depth = 0;
function hasFiles(event) {
return event.dataTransfer && Array.prototype.indexOf.call(
event.dataTransfer.types || [], "Files"
) !== -1;
}
zone.addEventListener("dragenter", function (event) {
if (!hasFiles(event)) return;
event.preventDefault();
depth += 1;
zone.classList.add("is-dropping");
});
zone.addEventListener("dragover", function (event) {
if (hasFiles(event)) event.preventDefault();
});
zone.addEventListener("dragleave", function () {
depth = Math.max(0, depth - 1);
if (depth === 0) zone.classList.remove("is-dropping");
});
zone.addEventListener("drop", function (event) {
if (!hasFiles(event)) return;
event.preventDefault();
depth = 0;
zone.classList.remove("is-dropping");
uploadFiles(event.dataTransfer.files);
});
/* Pasting a screenshot straight into the composer. Only files are taken;
pasted text must still behave as text. */
document.addEventListener("paste", function (event) {
var composer = event.target.closest("[data-composer-input]");
if (!composer || !event.clipboardData) return;
var files = Array.prototype.filter.call(
event.clipboardData.files || [], function (f) { return f && f.size; }
);
if (!files.length) return;
event.preventDefault();
uploadFiles(files);
});
}
window.lembas = { window.lembas = {
applyTheme: applyTheme, applyTheme: applyTheme,
toggleTheme: toggleTheme, toggleTheme: toggleTheme,
copyText: copyText, copyText: copyText,
scrollThread: scrollThread, scrollThread: scrollThread,
autosize: autosize autosize: autosize,
uploadFiles: uploadFiles
}; };
/* --- Wiring ------------------------------------------------------------ */ /* --- Wiring ------------------------------------------------------------ */
@@ -147,6 +235,7 @@
document.querySelectorAll("[data-autosize]").forEach(autosize); document.querySelectorAll("[data-autosize]").forEach(autosize);
scrollThread(true); scrollThread(true);
applyTheme(currentTheme()); applyTheme(currentTheme());
setupDropzone();
}); });
/* After any htmx swap: re-measure the composer and follow new content. */ /* After any htmx swap: re-measure the composer and follow new content. */
@@ -0,0 +1,38 @@
{% from "_macros.html" import icon %}
{#
A pending attachment in the composer, before the message is sent.
Carries its own id in a hidden input so the composer form submits it with the
message; removing the chip removes the input, which is all the bookkeeping
the client needs.
#}
<div class="chip" id="chip-{{ attachment.id }}">
<input type="hidden" name="file_ids" value="{{ attachment.id }}">
{% if attachment.is_image %}
<img class="chip__thumb" src="/api/files/{{ attachment.id }}/content" alt="">
{% else %}
<span class="chip__icon">
{{ icon("attach" if attachment.kind == "document" else "copy", "icon--sm") }}
</span>
{% endif %}
<span class="chip__body">
<span class="chip__name" title="{{ attachment.filename }}">{{ attachment.filename }}</span>
<span class="chip__meta">
{{ attachment.human_size }}
{%- if attachment.pages %} · {{ attachment.pages }} page{{ '' if attachment.pages == 1 else 's' }}{% endif %}
{%- if attachment.width %} · {{ attachment.width }}×{{ attachment.height }}{% endif %}
{%- if attachment.truncated %} · truncated{% endif %}
</span>
{% if attachment.extraction_error %}
<span class="chip__warning">{{ attachment.extraction_error }}</span>
{% endif %}
</span>
<button class="btn btn--icon btn--sm" type="button" aria-label="Remove {{ attachment.filename }}"
hx-delete="/api/files/{{ attachment.id }}"
hx-target="#chip-{{ attachment.id }}" hx-swap="outerHTML">
{{ icon("x", "icon--sm") }}
</button>
</div>
@@ -0,0 +1,17 @@
{% from "_macros.html" import icon %}
{#
A rejected upload. Rendered in place of a chip so the reason is visible in
the composer rather than only in the network tab. Dismissed by hand; it
carries no hidden input, so it cannot be submitted with the message.
#}
<div class="chip chip--error">
<span class="chip__icon">{{ icon("warning", "icon--sm") }}</span>
<span class="chip__body">
<span class="chip__name">{{ filename }}</span>
<span class="chip__warning">{{ error }}</span>
</span>
<button class="btn btn--icon btn--sm" type="button" aria-label="Dismiss"
onclick="this.closest('.chip').remove()">
{{ icon("x", "icon--sm") }}
</button>
</div>
+38 -1
View File
@@ -38,6 +38,41 @@
{% endif %} {% endif %}
</header> </header>
{% if message.attachments %}
{# Above the text, matching the order they were added and the order the
model receives them. #}
<div class="attachments">
{% for attachment in message.attachments %}
{% if attachment.is_image %}
<a class="attachments__image" href="/api/files/{{ attachment.id }}/content"
target="_blank" rel="noopener">
<img src="/api/files/{{ attachment.id }}/content" alt="{{ attachment.filename }}"
loading="lazy" width="{{ attachment.width }}" height="{{ attachment.height }}">
</a>
{% else %}
<div class="attachments__doc">
{{ icon("attach", "icon--sm") }}
<span class="attachments__doc-body">
<a href="/api/files/{{ attachment.id }}/content">{{ attachment.filename }}</a>
<span class="chip__meta">
{{ attachment.human_size }}
{%- if attachment.pages %} · {{ attachment.pages }} page{{ '' if attachment.pages == 1 else 's' }}{% endif %}
{%- if attachment.truncated %} · truncated{% endif %}
{%- if attachment.extracted_text %}
· <a href="/api/files/{{ attachment.id }}/text" target="_blank"
rel="noopener">view extracted text</a>
{%- endif %}
</span>
{% if attachment.extraction_error %}
<span class="chip__warning">{{ attachment.extraction_error }}</span>
{% endif %}
</span>
</div>
{% endif %}
{% endfor %}
</div>
{% endif %}
{% if streaming %} {% if streaming %}
{# Reasoning arrives before the answer, so this block sits above it. It {# Reasoning arrives before the answer, so this block sits above it. It
starts open (watching a model think is the point) and the :has() rule starts open (watching a model think is the point) and the :has() rule
@@ -92,9 +127,11 @@
{% endif %} {% endif %}
{% elif message.role == "assistant" %} {% elif message.role == "assistant" %}
<div class="msg__body">{{ body_html|safe }}</div> <div class="msg__body">{{ body_html|safe }}</div>
{% else %} {% elif message.content %}
<div class="msg__body msg__body--plain">{{ message.content }}</div> <div class="msg__body msg__body--plain">{{ message.content }}</div>
{% endif %} {% endif %}
{# An attachment-only turn has no text; rendering the bubble anyway would
leave an empty box under the file. #}
{% if not streaming %} {% if not streaming %}
<footer class="msg__actions"> <footer class="msg__actions">
+45 -1
View File
@@ -164,16 +164,45 @@
</div> </div>
</div> </div>
<div class="composer"> <div class="composer" {% if can.get("files.upload") %}data-dropzone{% endif %}>
{% if can.get("files.upload") %}
{# Uploads go up as soon as a file is chosen, so the chip (and any
rejection) appears immediately rather than at send time. The chips
carry hidden inputs, which is how the ids reach the message POST. #}
<form id="upload-form" hx-post="/api/files?chat_id={{ chat.id }}"
hx-target="#attachments" hx-swap="beforeend"
hx-encoding="multipart/form-data"
hx-on::after-request="this.reset()">
<input class="visually-hidden" type="file" name="file" id="file-input"
multiple accept="image/*,.pdf,.txt,.md,.csv,.json,.py,.js,.ts,.rs,.go,.sh,.sql,.yaml,.yml,.toml,.log"
onchange="window.lembas.uploadFiles(this.files); this.value = ''">
</form>
{% endif %}
<div class="composer__attachments" id="attachments"></div>
<form class="composer__form" <form class="composer__form"
hx-post="/api/chats/{{ chat.id }}/messages" hx-post="/api/chats/{{ chat.id }}/messages"
hx-target="#thread" hx-swap="beforeend" hx-target="#thread" hx-swap="beforeend"
hx-on::after-request="if (event.detail.successful) { hx-on::after-request="if (event.detail.successful) {
this.reset(); this.reset();
document.getElementById('attachments').replaceChildren();
const t = this.querySelector('textarea'); const t = this.querySelector('textarea');
window.lembas.autosize(t); window.lembas.autosize(t);
window.lembas.scrollThread(true); window.lembas.scrollThread(true);
}"> }">
{# The chips live outside this form, so their hidden inputs are pulled
in explicitly at submit time. #}
<div hx-include="#attachments" hidden></div>
{% if can.get("files.upload") %}
<button class="btn btn--icon composer__attach" type="button"
aria-label="Attach a file" title="Attach a file"
onclick="document.getElementById('file-input').click()">
{{ icon("attach") }}
</button>
{% endif %}
<textarea class="composer__input" name="content" rows="1" <textarea class="composer__input" name="content" rows="1"
data-autosize data-max-height="320" data-composer-input data-autosize data-max-height="320" data-composer-input
placeholder="Send a message…" aria-label="Message"></textarea> placeholder="Send a message…" aria-label="Message"></textarea>
@@ -181,9 +210,24 @@
{{ icon("send", "icon--sm") }} {{ icon("send", "icon--sm") }}
</button> </button>
</form> </form>
<p class="composer__hint"> <p class="composer__hint">
Enter to send, Shift+Enter for a new line. Enter to send, Shift+Enter for a new line.
{% if can.get("files.upload") %}
Drag files in, or paste an image.
{% if current_model and not current_model.capabilities_json.get("vision") %}
<strong>{{ current_model.label }} has no vision</strong>, so images
will not be sent — documents still will.
{% endif %}
{% endif %}
</p> </p>
{% if can.get("files.upload") %}
<div class="dropzone-overlay" aria-hidden="true">
{{ icon("attach", "icon--lg") }}
<span>Drop to attach</span>
</div>
{% endif %}
</div> </div>
{% endif %} {% endif %}
</main> </main>
+496
View File
@@ -0,0 +1,496 @@
"""Attachments: upload validation, extraction, and how they reach the model."""
from __future__ import annotations
import io
from datetime import timedelta
import pytest
from fastapi.testclient import TestClient
from PIL import Image
from sqlalchemy import select
from lembas.db.models import Attachment, Chat, Connection, Message, Model
from lembas.services import chat as chat_service
from lembas.services import files as files_service
from lembas.services import settings_store
from lembas.services.crypto import encrypt
# --- Fixtures ----------------------------------------------------------------
def png_bytes(width: int = 40, height: int = 30, mode: str = "RGB") -> bytes:
buffer = io.BytesIO()
Image.new(mode, (width, height), "red").save(buffer, format="PNG")
return buffer.getvalue()
def jpeg_bytes(width: int = 40, height: int = 30) -> bytes:
buffer = io.BytesIO()
Image.new("RGB", (width, height), "blue").save(buffer, format="JPEG")
return buffer.getvalue()
def pdf_bytes(pages: list[str]) -> bytes:
"""A real PDF with a text layer, built with pypdf + reportlab-free drawing.
pypdf cannot author text, so the file is assembled by hand. It is minimal
but genuinely parseable, which is the point -- a fake would not exercise
extraction at all.
"""
objects: list[bytes] = []
def stream_for(text: str) -> bytes:
escaped = text.replace("\\", r"\\").replace("(", r"\(").replace(")", r"\)")
return f"BT /F1 12 Tf 72 720 Td ({escaped}) Tj ET".encode("latin-1")
page_ids = [4 + i * 2 for i in range(len(pages))]
kids = " ".join(f"{pid} 0 R" for pid in page_ids)
objects.append(b"<< /Type /Catalog /Pages 2 0 R >>")
objects.append(f"<< /Type /Pages /Kids [{kids}] /Count {len(pages)} >>".encode())
objects.append(b"<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica >>")
for index, text in enumerate(pages):
content = stream_for(text)
objects.append(
f"<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] "
f"/Resources << /Font << /F1 3 0 R >> >> "
f"/Contents {page_ids[index] + 1} 0 R >>".encode()
)
objects.append(
b"<< /Length "
+ str(len(content)).encode()
+ b" >>\nstream\n"
+ content
+ b"\nendstream"
)
out = bytearray(b"%PDF-1.4\n")
offsets = [0]
for number, body in enumerate(objects, start=1):
offsets.append(len(out))
out += f"{number} 0 obj\n".encode() + body + b"\nendobj\n"
xref_at = len(out)
out += f"xref\n0 {len(objects) + 1}\n".encode()
out += b"0000000000 65535 f \n"
for offset in offsets[1:]:
out += f"{offset:010d} 00000 n \n".encode()
out += (
f"trailer\n<< /Size {len(objects) + 1} /Root 1 0 R >>\nstartxref\n{xref_at}\n%%EOF".encode()
)
return bytes(out)
@pytest.fixture
def chat_with_model(client: TestClient, db, registered):
"""A chat whose model has vision turned on."""
connection = Connection(name="T", base_url="http://127.0.0.1:1", api_key_encrypted=encrypt(""))
db.add(connection)
db.commit()
db.add(
Model(
connection_id=connection.id,
model_id="seeing-model",
capabilities_json={"vision": True},
)
)
db.commit()
chat_id = client.post("/api/chats").headers["HX-Redirect"].rsplit("/", 1)[1]
return chat_id
# --- Type detection and processing -------------------------------------------
def test_png_is_recognised_as_an_image():
prepared = files_service.prepare(png_bytes(), "photo.png")
assert prepared.kind == "image"
assert prepared.width == 40 and prepared.height == 30
def test_large_images_are_downscaled():
"""A phone photo is megabytes of base64 and a large slice of the context."""
prepared = files_service.prepare(jpeg_bytes(4000, 3000), "big.jpg")
assert max(prepared.width, prepared.height) == files_service.MAX_IMAGE_EDGE
assert prepared.height == int(3000 * (files_service.MAX_IMAGE_EDGE / 4000))
def test_small_images_are_left_alone():
prepared = files_service.prepare(jpeg_bytes(100, 80), "small.jpg")
assert (prepared.width, prepared.height) == (100, 80)
def test_transparent_images_stay_png():
prepared = files_service.prepare(png_bytes(mode="RGBA"), "logo.png")
assert prepared.media_type in ("image/png", "image/jpeg")
def test_a_file_lying_about_its_type_is_judged_by_its_bytes():
"""The extension says PNG; the content is text, and content wins."""
prepared = files_service.prepare(b"just some words", "trick.png")
assert prepared.kind == "text"
def test_pdf_text_is_extracted():
prepared = files_service.prepare(pdf_bytes(["Lembas keeps a traveller going."]), "doc.pdf")
assert prepared.kind == "document"
assert prepared.pages == 1
assert "Lembas keeps a traveller going." in prepared.extracted_text
def test_multi_page_pdfs_are_labelled_by_page():
prepared = files_service.prepare(pdf_bytes(["First page here", "Second page here"]), "d.pdf")
assert prepared.pages == 2
assert "[page 1]" in prepared.extracted_text
assert "[page 2]" in prepared.extracted_text
def test_a_pdf_with_no_text_layer_says_so():
"""A scan otherwise looks like the model simply ignored the document."""
prepared = files_service.prepare(pdf_bytes([" "]), "scan.pdf")
assert prepared.extraction_error
assert "scanned" in prepared.extraction_error.lower()
def test_a_corrupt_pdf_is_stored_with_an_error_not_rejected():
prepared = files_service.prepare(b"%PDF-1.4\nthis is not really a pdf", "broken.pdf")
assert prepared.kind == "document"
assert prepared.extraction_error
def test_text_files_are_decoded():
prepared = files_service.prepare(b"line one\nline two", "notes.txt")
assert prepared.kind == "text"
assert prepared.extracted_text == "line one\nline two"
def test_source_files_keep_a_sensible_media_type():
assert files_service.prepare(b"print('hi')", "x.py").media_type == "text/x-python"
def test_binary_files_are_rejected():
with pytest.raises(files_service.FileError):
files_service.prepare(b"\x00\x01\x02\x03" * 100, "mystery.bin")
def test_empty_files_are_rejected():
with pytest.raises(files_service.FileError):
files_service.prepare(b"", "empty.txt")
def test_oversized_files_are_rejected():
with pytest.raises(files_service.FileError) as caught:
files_service.prepare(b"x" * (files_service.MAX_UPLOAD_BYTES + 1), "huge.txt")
assert "MB" in str(caught.value)
def test_extracted_text_is_capped(monkeypatch):
monkeypatch.setattr(files_service, "MAX_EXTRACTED_CHARS", 50)
prepared = files_service.prepare(b"x" * 500, "long.txt")
assert len(prepared.extracted_text) == 50
assert prepared.truncated is True
# --- Path safety -------------------------------------------------------------
@pytest.mark.parametrize(
"name", ["../../etc/passwd", "..\\windows", "/etc/passwd", ".hidden", ""]
)
def test_stored_path_refuses_traversal(name):
assert files_service.stored_path(name) is None
def test_stored_names_are_random_not_the_uploaders(client: TestClient, db, registered):
response = client.post(
"/api/files", files={"file": ("../../evil.txt", b"content", "text/plain")}
)
assert response.status_code == 200
attachment = db.scalar(select(Attachment))
assert "/" not in attachment.stored_name
assert ".." not in attachment.stored_name
# The display name is kept, but only as a label.
assert attachment.filename == "evil.txt"
# --- Upload through the API --------------------------------------------------
def test_upload_returns_a_chip(client: TestClient, db, registered):
response = client.post(
"/api/files", files={"file": ("notes.txt", b"hello there", "text/plain")}
)
assert response.status_code == 200
assert "notes.txt" in response.text
assert 'name="file_ids"' in response.text
def test_a_rejected_upload_returns_a_readable_error(client: TestClient, db, registered):
response = client.post(
"/api/files", files={"file": ("bad.bin", b"\x00\x01" * 500, "application/octet-stream")}
)
assert response.status_code == 200
assert "chip--error" in response.text
assert db.scalar(select(Attachment)) is None
def test_uploading_needs_permission(client: TestClient, db, registered):
client.post("/auth/logout", follow_redirects=False)
client.post(
"/auth/register",
data={"name": "Sam", "email": "sam@shire.test", "password": "potatoes-po-ta-toes"},
follow_redirects=False,
)
settings_store.update(db, {"default_permissions": {"files.upload": False}})
response = client.post("/api/files", files={"file": ("a.txt", b"hi", "text/plain")})
assert response.status_code == 403
def test_you_cannot_fetch_someone_elses_file(client: TestClient, db, registered):
client.post("/api/files", files={"file": ("secret.txt", b"mine", "text/plain")})
attachment = db.scalar(select(Attachment))
client.post("/auth/logout", follow_redirects=False)
client.post(
"/auth/register",
data={"name": "Sam", "email": "sam@shire.test", "password": "potatoes-po-ta-toes"},
follow_redirects=False,
)
assert client.get(f"/api/files/{attachment.id}/content").status_code == 404
def test_non_images_are_served_as_downloads(client: TestClient, db, registered):
"""An uploaded .html served inline would run in this origin."""
client.post("/api/files", files={"file": ("page.html", b"<b>hi</b>", "text/html")})
attachment = db.scalar(select(Attachment))
response = client.get(f"/api/files/{attachment.id}/content")
assert "attachment;" in response.headers["content-disposition"]
assert response.headers["content-type"] == "application/octet-stream"
assert response.headers["x-content-type-options"] == "nosniff"
def test_images_are_served_inline(client: TestClient, db, registered):
client.post("/api/files", files={"file": ("p.png", png_bytes(), "image/png")})
attachment = db.scalar(select(Attachment))
response = client.get(f"/api/files/{attachment.id}/content")
assert "inline;" in response.headers["content-disposition"]
def test_an_unsent_attachment_can_be_removed(client: TestClient, db, registered):
client.post("/api/files", files={"file": ("a.txt", b"hi", "text/plain")})
attachment = db.scalar(select(Attachment))
stored = files_service.stored_path(attachment.stored_name)
assert client.delete(f"/api/files/{attachment.id}").status_code == 200
assert db.scalar(select(Attachment)) is None
assert not stored.exists()
def test_a_sent_attachment_cannot_be_removed(client: TestClient, db, chat_with_model):
"""Deleting it would rewrite a conversation the user has already read."""
client.post("/api/files", files={"file": ("a.txt", b"hi", "text/plain")})
attachment = db.scalar(select(Attachment))
client.post(
f"/api/chats/{chat_with_model}/messages",
data={"content": "look", "file_ids": [attachment.id]},
)
assert client.delete(f"/api/files/{attachment.id}").status_code == 409
# --- Attaching to a message --------------------------------------------------
def test_sending_binds_the_attachment_to_the_message(client: TestClient, db, chat_with_model):
client.post("/api/files", files={"file": ("a.txt", b"hi", "text/plain")})
attachment = db.scalar(select(Attachment))
client.post(
f"/api/chats/{chat_with_model}/messages",
data={"content": "have a look", "file_ids": [attachment.id]},
)
db.refresh(attachment)
message = db.scalar(select(Message).where(Message.role == "user"))
assert attachment.message_id == message.id
def test_a_message_with_only_an_attachment_is_accepted(client: TestClient, db, chat_with_model):
""""Here, look at this" with no words is a legitimate turn."""
client.post("/api/files", files={"file": ("a.png", png_bytes(), "image/png")})
attachment = db.scalar(select(Attachment))
response = client.post(
f"/api/chats/{chat_with_model}/messages",
data={"content": "", "file_ids": [attachment.id]},
)
assert response.status_code == 200
assert db.scalar(select(Message).where(Message.role == "user")) is not None
def test_a_truly_empty_message_is_still_ignored(client: TestClient, db, chat_with_model):
response = client.post(f"/api/chats/{chat_with_model}/messages", data={"content": " "})
assert response.status_code == 204
assert db.scalar(select(Message)) is None
def test_you_cannot_attach_someone_elses_file(client: TestClient, db, chat_with_model):
"""A forged id must not pull another user's file into a conversation."""
client.post("/api/files", files={"file": ("mine.txt", b"secret", "text/plain")})
stolen = db.scalar(select(Attachment))
client.post("/auth/logout", follow_redirects=False)
client.post(
"/auth/register",
data={"name": "Sam", "email": "sam@shire.test", "password": "potatoes-po-ta-toes"},
follow_redirects=False,
)
their_chat = client.post("/api/chats").headers["HX-Redirect"].rsplit("/", 1)[1]
client.post(
f"/api/chats/{their_chat}/messages",
data={"content": "gimme", "file_ids": [stolen.id]},
)
db.refresh(stolen)
assert stolen.message_id is None
# --- What reaches the model --------------------------------------------------
def _user_message(db, chat_id: str) -> Message:
return db.scalar(
select(Message).where(Message.chat_id == chat_id, Message.role == "user")
)
def test_images_become_multimodal_parts_for_a_vision_model(client: TestClient, db, chat_with_model):
client.post("/api/files", files={"file": ("p.png", png_bytes(), "image/png")})
attachment = db.scalar(select(Attachment))
client.post(
f"/api/chats/{chat_with_model}/messages",
data={"content": "what is this?", "file_ids": [attachment.id]},
)
chat = db.get(Chat, chat_with_model)
payload = chat_service.build_request(db, chat)
content = payload["messages"][0]["content"]
assert isinstance(content, list)
assert content[0] == {"type": "text", "text": "what is this?"}
assert content[1]["type"] == "image_url"
assert content[1]["image_url"]["url"].startswith("data:image/")
def test_images_are_withheld_from_a_model_without_vision(client: TestClient, db, chat_with_model):
"""Most endpoints reject the whole request rather than ignoring the image."""
model = db.scalar(select(Model))
model.capabilities_json = {"vision": False}
db.commit()
client.post("/api/files", files={"file": ("p.png", png_bytes(), "image/png")})
attachment = db.scalar(select(Attachment))
client.post(
f"/api/chats/{chat_with_model}/messages",
data={"content": "what is this?", "file_ids": [attachment.id]},
)
chat = db.get(Chat, chat_with_model)
content = chat_service.build_request(db, chat)["messages"][0]["content"]
assert isinstance(content, str)
assert content == "what is this?"
def test_a_plain_turn_stays_a_plain_string(client: TestClient, db, chat_with_model):
"""The list form is a reliable 400 from endpoints that do not implement it."""
client.post(f"/api/chats/{chat_with_model}/messages", data={"content": "just words"})
chat = db.get(Chat, chat_with_model)
assert chat_service.build_request(db, chat)["messages"][0]["content"] == "just words"
def test_document_text_is_prepended_in_tags(client: TestClient, db, chat_with_model):
client.post(
"/api/files", files={"file": ("report.txt", b"Quarterly results were good.", "text/plain")}
)
attachment = db.scalar(select(Attachment))
client.post(
f"/api/chats/{chat_with_model}/messages",
data={"content": "summarise this", "file_ids": [attachment.id]},
)
chat = db.get(Chat, chat_with_model)
content = chat_service.build_request(db, chat)["messages"][0]["content"]
assert '<document name="report.txt">' in content
assert "Quarterly results were good." in content
# The question comes after the material it refers to.
assert content.index("</document>") < content.index("summarise this")
def test_documents_reach_a_model_without_vision(client: TestClient, db, chat_with_model):
model = db.scalar(select(Model))
model.capabilities_json = {}
db.commit()
client.post("/api/files", files={"file": ("notes.txt", b"important detail", "text/plain")})
attachment = db.scalar(select(Attachment))
client.post(
f"/api/chats/{chat_with_model}/messages",
data={"content": "read it", "file_ids": [attachment.id]},
)
chat = db.get(Chat, chat_with_model)
content = chat_service.build_request(db, chat)["messages"][0]["content"]
assert "important detail" in content
def test_truncation_is_declared_to_the_model(client: TestClient, db, chat_with_model, monkeypatch):
"""A model asked about page 400 should be able to say it did not see it."""
monkeypatch.setattr(files_service, "MAX_EXTRACTED_CHARS", 20)
client.post("/api/files", files={"file": ("big.txt", b"y" * 200, "text/plain")})
attachment = db.scalar(select(Attachment))
client.post(
f"/api/chats/{chat_with_model}/messages",
data={"content": "read", "file_ids": [attachment.id]},
)
chat = db.get(Chat, chat_with_model)
assert "(truncated)" in chat_service.build_request(db, chat)["messages"][0]["content"]
def test_an_attachment_only_turn_still_reaches_the_model(client: TestClient, db, chat_with_model):
client.post("/api/files", files={"file": ("p.png", png_bytes(), "image/png")})
attachment = db.scalar(select(Attachment))
client.post(
f"/api/chats/{chat_with_model}/messages", data={"content": "", "file_ids": [attachment.id]}
)
chat = db.get(Chat, chat_with_model)
messages = chat_service.build_request(db, chat)["messages"]
assert len(messages) == 1
assert messages[0]["content"][0]["type"] == "image_url"
# --- Housekeeping ------------------------------------------------------------
def test_orphans_are_swept(client: TestClient, db, registered):
client.post("/api/files", files={"file": ("a.txt", b"hi", "text/plain")})
attachment = db.scalar(select(Attachment))
path = files_service.stored_path(attachment.stored_name)
assert files_service.sweep_orphans(db, timedelta(hours=24)) == 0 # too new
assert files_service.sweep_orphans(db, timedelta(seconds=-1)) == 1
assert db.scalar(select(Attachment)) is None
assert not path.exists()
def test_sent_attachments_are_never_swept(client: TestClient, db, chat_with_model):
client.post("/api/files", files={"file": ("a.txt", b"hi", "text/plain")})
attachment = db.scalar(select(Attachment))
client.post(
f"/api/chats/{chat_with_model}/messages",
data={"content": "keep", "file_ids": [attachment.id]},
)
assert files_service.sweep_orphans(db, timedelta(seconds=-1)) == 0
assert db.scalar(select(Attachment)) is not None
def test_deleting_a_message_deletes_its_attachments(client: TestClient, db, chat_with_model):
client.post("/api/files", files={"file": ("a.txt", b"hi", "text/plain")})
attachment = db.scalar(select(Attachment))
client.post(
f"/api/chats/{chat_with_model}/messages",
data={"content": "x", "file_ids": [attachment.id]},
)
client.delete(f"/api/chats/{chat_with_model}")
assert db.scalar(select(Attachment)) is None