Knowledge bases, and a file input that lines up

**Bases.** Documents now live in named collections rather than one flat pile,
and a chat can be pointed at particular ones — "answer from the contracts
folder" is a different question from "answer from everything I have ever
uploaded". A chat with none attached still searches everything its owner can
see, because empty means unscoped, not empty.

The harness names the attached bases. Without that the model cannot tell "there
is nothing about this" from "I am only allowed to see one folder", and it
phrases a miss as the former.

**Sharing moves to the base.** A document is visible to whoever can see the base
it lives in, so `Document` is gone from the shareable types and
`documents.visible()` filters through `base_id`. "This folder is the team's" is
the granularity people think in; per-document grants meant answering "who can
see this?" by checking every file. Moving a document between bases changes who
can see it, so the destination has to be one you own.

`Document.base_id` is nullable only because the column had to be added to a
table that already had rows. `sweep_unfiled()` runs at startup beside the
orphaned-upload sweep and files anything predating bases into its owner's
default, which is what makes "always set" true everywhere else.

**The file input.** `.input` gave it a fixed height and horizontal padding, so
the browser's own button sat hard against the left edge while the filename
floated off the centre line. A file input is two controls in one box and
neither inherits anything useful, so it gets its own rule: no horizontal
padding, the button sized to `--control-h` with the divider that separates it,
and the text centred with line-height rather than flexbox, which file inputs do
not lay out reliably.

437 tests, ruff clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
2026-07-21 20:00:15 +02:00
parent 1eba860d39
commit 35b9d8c8d2
22 changed files with 809 additions and 137 deletions
+20
View File
@@ -484,6 +484,26 @@ async def update_chat(request: Request, db: Db, user: RequiredUser, chat_id: str
)
chat.system_prompt = str(form["system_prompt"]).strip()[:8000]
if "knowledge_base_ids" in form:
# Sent as a single field even when empty, so that clearing every box
# actually clears the attachment -- absent checkboxes carry no signal of
# their own, which is the same trap update_chat exists to avoid.
from lembas.db.models import KnowledgeBase
from lembas.services.library import documents as documents_service
wanted = [value for value in form.getlist("knowledge_base_ids") if value]
chat.knowledge_bases = (
list(
db.scalars(
documents_service.visible_bases(db, user).where(
KnowledgeBase.id.in_(wanted)
)
)
)
if wanted
else []
)
submitted_params = {name: form[name] for name in _PARAM_RANGES if name in form}
if submitted_params:
if not allowed.get("chat.params"):
+149 -25
View File
@@ -27,6 +27,7 @@ from lembas.db.models import (
PRINCIPAL_USER,
Document,
Group,
KnowledgeBase,
Note,
Skill,
SkillRevision,
@@ -92,33 +93,54 @@ async def library_home(user: RequiredUser):
# --- Knowledge ---------------------------------------------------------------
# Route order matters: /library/knowledge/document/{id} must be registered
# before /library/knowledge/{base_id}, or "document" is parsed as a base id.
# FastAPI matches in registration order and this has bitten before.
@router.get("/library/knowledge")
async def knowledge_list(
request: Request, db: Db, user: RequiredUser, q: str = "", page: int = 1, saved: str = ""
):
if q.strip():
# Search returns best-match order and its own limit, so it is not paged.
rows = documents_service.search(db, user, q, limit=PAGE_SIZE)
pager = {"page": 1, "pages": 1, "total": len(rows)}
else:
rows, pager = _page(
db, documents_service.visible(db, user).order_by(Document.created_at.desc()), page
async def knowledge_list(request: Request, db: Db, user: RequiredUser, error: str = ""):
"""The bases, not the documents. A library is a set of places first."""
bases = list(db.scalars(documents_service.visible_bases(db, user).order_by(KnowledgeBase.name)))
counts = {
base.id: db.scalar(
select(func.count()).select_from(Document).where(Document.base_id == base.id)
)
or 0
for base in bases
}
return render(
request,
"library/knowledge.html",
{
"section": "knowledge",
"documents": rows,
"q": q,
"pager": pager,
"saved": saved,
"bases": bases,
"counts": counts,
"error": error,
**sidebar_context(db, user),
},
)
@router.get("/library/knowledge/{document_id}")
@router.post("/api/library/bases")
async def create_base(
db: Db, user: RequiredUser, name: str = Form(""), description: str = Form("")
) -> Response:
try:
base = documents_service.create_base(
db, owner=user, name=name, description=description
)
except ValueError as exc:
from urllib.parse import quote
return RedirectResponse(
f"/library/knowledge?error={quote(str(exc))}",
status_code=status.HTTP_303_SEE_OTHER,
)
return RedirectResponse(
f"/library/knowledge/{base.id}", status_code=status.HTTP_303_SEE_OTHER
)
@router.get("/library/knowledge/document/{document_id}")
async def knowledge_detail(request: Request, db: Db, user: RequiredUser, document_id: str):
document = documents_service.get(db, document_id, user)
if document is None:
@@ -129,38 +151,129 @@ async def knowledge_detail(request: Request, db: Db, user: RequiredUser, documen
{
"section": "knowledge",
"document": document,
**_shared_context(db, user, document),
"is_owner": sharing.can_write(document, user),
# Only bases this person owns: moving a document into one they can
# merely read would hand it to that base's owner.
"user_bases": list(
db.scalars(
select(KnowledgeBase)
.where(KnowledgeBase.owner_id == user.id)
.order_by(KnowledgeBase.name)
)
),
**sidebar_context(db, user),
},
)
@router.get("/library/knowledge/{base_id}")
async def base_detail(
request: Request, db: Db, user: RequiredUser, base_id: str, q: str = "", page: int = 1
):
base = documents_service.get_base(db, base_id, user)
if base is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, "That knowledge base is not available.")
if q.strip():
rows = documents_service.search(db, user, q, limit=PAGE_SIZE, base_ids=[base.id])
pager = {"page": 1, "pages": 1, "total": len(rows)}
else:
rows, pager = _page(
db,
documents_service.visible(db, user, base_ids=[base.id]).order_by(
Document.created_at.desc()
),
page,
)
return render(
request,
"library/base_detail.html",
{
"section": "knowledge",
"base": base,
"documents": rows,
"q": q,
"pager": pager,
**_shared_context(db, user, base),
**sidebar_context(db, user),
},
)
@router.post("/api/library/bases/{base_id}")
async def update_base(request: Request, db: Db, user: RequiredUser, base_id: str) -> Response:
base = documents_service.get_base(db, base_id, user)
if base is None:
raise HTTPException(status.HTTP_404_NOT_FOUND, "That knowledge base is not available.")
if not sharing.can_write(base, user):
raise HTTPException(status.HTTP_403_FORBIDDEN, "That base is not yours to change.")
form = await request.form()
name = " ".join(str(form.get("name", "")).split())[:200]
if name:
base.name = name
base.description = str(form.get("description", "")).strip()[:2000]
db.commit()
_apply_shares(db, user, base, form)
return RedirectResponse(
f"/library/knowledge/{base.id}", status_code=status.HTTP_303_SEE_OTHER
)
@router.post("/api/library/bases/{base_id}/delete")
async def delete_base(db: Db, user: RequiredUser, base_id: str) -> Response:
base = documents_service.get_base(db, base_id, user)
if base is None or not sharing.can_write(base, user):
raise HTTPException(status.HTTP_404_NOT_FOUND, "That knowledge base is not available.")
documents_service.delete_base(db, base)
return RedirectResponse("/library/knowledge", status_code=status.HTTP_303_SEE_OTHER)
@router.post("/api/library/documents")
async def upload_document(
db: Db, user: RequiredUser, file: UploadFile = File(...), title: str = Form("")
db: Db,
user: RequiredUser,
file: UploadFile = File(...),
title: str = Form(""),
base_id: str = Form(""),
) -> Response:
base = documents_service.get_base(db, base_id, user) if base_id else None
if base is not None and not sharing.can_write(base, user):
raise HTTPException(status.HTTP_403_FORBIDDEN, "That base is not yours to add to.")
payload = await file.read(files_service.MAX_UPLOAD_BYTES + 1)
try:
document = documents_service.store_upload(
db, owner=user, payload=payload, filename=file.filename or "file", title=title
db,
owner=user,
payload=payload,
filename=file.filename or "file",
title=title,
base=base,
)
except files_service.FileError as exc:
raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc
return RedirectResponse(
f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER
f"/library/knowledge/{document.base_id}", status_code=status.HTTP_303_SEE_OTHER
)
@router.post("/api/library/documents/link")
async def save_link(db: Db, user: RequiredUser, url: str = Form(...)) -> Response:
async def save_link(
db: Db, user: RequiredUser, url: str = Form(...), base_id: str = Form("")
) -> Response:
base = documents_service.get_base(db, base_id, user) if base_id else None
if base is not None and not sharing.can_write(base, user):
raise HTTPException(status.HTTP_403_FORBIDDEN, "That base is not yours to add to.")
config = settings_store.search(db)
try:
page = await fetch(url, allow_private=bool(config.get("allow_private_fetch")))
except FetchError as exc:
raise HTTPException(status.HTTP_400_BAD_REQUEST, exc.message) from exc
document = documents_service.store_page(db, owner=user, page=page)
document = documents_service.store_page(db, owner=user, page=page, base=base)
return RedirectResponse(
f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER
f"/library/knowledge/{document.base_id}", status_code=status.HTTP_303_SEE_OTHER
)
@@ -177,10 +290,18 @@ async def update_document(
form = await request.form()
document.title = str(form.get("title", document.title)).strip()[:300] or document.title
document.description = str(form.get("description", "")).strip()[:2000]
# Moving between bases changes who can see it, which is the whole point of
# bases -- so the destination has to be one this person can write to.
wanted = str(form.get("base_id", "")).strip()
if wanted and wanted != document.base_id:
destination = documents_service.get_base(db, wanted, user)
if destination is not None and sharing.can_write(destination, user):
document.base_id = destination.id
db.commit()
_apply_shares(db, user, document, form)
return RedirectResponse(
f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER
f"/library/knowledge/document/{document.id}", status_code=status.HTTP_303_SEE_OTHER
)
@@ -189,8 +310,11 @@ async def delete_document(db: Db, user: RequiredUser, document_id: str) -> Respo
document = documents_service.get(db, document_id, user)
if document is None or not sharing.can_write(document, user):
raise HTTPException(status.HTTP_404_NOT_FOUND, "That document is not available.")
base_id = document.base_id
documents_service.delete(db, document)
return RedirectResponse("/library/knowledge", status_code=status.HTTP_303_SEE_OTHER)
return RedirectResponse(
f"/library/knowledge/{base_id}", status_code=status.HTTP_303_SEE_OTHER
)
@router.get("/api/library/documents/{document_id}/content")
+15 -3
View File
@@ -8,11 +8,12 @@ from sqlalchemy import select
from sqlalchemy.orm import Session as DBSession
from lembas.api.deps import Db, RequiredUser
from lembas.db.models import Chat, Folder, Message, User
from lembas.db.models import Chat, Folder, KnowledgeBase, Message, User
from lembas.security import permissions
from lembas.services import audio as audio_service
from lembas.services import chat as chat_service
from lembas.services import settings_store
from lembas.services.library import documents as documents_service
from lembas.services.markdown import render_markdown
from lembas.web.templating import STATIC_DIR, render
@@ -42,6 +43,19 @@ def _chat_context(db: DBSession, user: User, chat: Chat | None) -> dict:
# may not be the model the chat is set to now. Keyed by model_id, the
# denormalised value stored on each message.
"models_by_id": {m.model_id: m for m in models},
# Offered in the chat settings panel so a conversation can be pointed at
# particular bases. Empty when the reader has none, and the panel then
# shows nothing rather than an empty control.
"knowledge_bases": (
list(
db.scalars(
documents_service.visible_bases(db, user).order_by(KnowledgeBase.name)
)
)
if permissions.has(db, user, "library.use")
else []
),
"attached_base_ids": [base.id for base in chat.knowledge_bases] if chat else [],
**audio_service.template_flags(db, user),
}
@@ -223,8 +237,6 @@ async def chat_detail(request: Request, db: Db, user: RequiredUser, chat_id: str
if current is not None and (current.system_prompt or "").strip():
inherited, inherited_from = current.system_prompt.strip(), "model"
else:
from lembas.services import settings_store
instance_prompt = (settings_store.get(db, "system_prompt") or "").strip()
if instance_prompt:
inherited, inherited_from = instance_prompt, "instance"
+6 -2
View File
@@ -26,17 +26,19 @@ from lembas.db.models.library import (
AUTHOR_USER,
PRINCIPAL_GROUP,
PRINCIPAL_USER,
RESOURCE_DOCUMENT,
RESOURCE_BASE,
RESOURCE_NOTE,
RESOURCE_SKILL,
SOURCE_LINK,
SOURCE_UPLOAD,
Document,
KnowledgeBase,
Memory,
Note,
Share,
Skill,
SkillRevision,
chat_knowledge_bases,
)
from lembas.db.models.setting import Setting
from lembas.db.models.user import (
@@ -57,7 +59,7 @@ __all__ = [
"KIND_TEXT",
"PRINCIPAL_GROUP",
"PRINCIPAL_USER",
"RESOURCE_DOCUMENT",
"RESOURCE_BASE",
"RESOURCE_NOTE",
"RESOURCE_SKILL",
"ROLE_ADMIN",
@@ -73,6 +75,7 @@ __all__ = [
"Document",
"Folder",
"Group",
"KnowledgeBase",
"Memory",
"Message",
"Model",
@@ -83,6 +86,7 @@ __all__ = [
"Skill",
"SkillRevision",
"User",
"chat_knowledge_bases",
"model_groups",
"user_groups",
]
+12 -1
View File
@@ -2,7 +2,7 @@
from __future__ import annotations
from typing import Any
from typing import TYPE_CHECKING, Any
from sqlalchemy import Boolean, ForeignKey, Integer, String, Text
from sqlalchemy.orm import Mapped, mapped_column, relationship
@@ -10,6 +10,12 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship
from lembas.db.base import Base, Timestamps, UUIDPrimaryKey
from lembas.db.types import JSONDict, JSONList
if TYPE_CHECKING:
# Annotation only; SQLAlchemy resolves the name through its own registry at
# runtime, so there is no import cycle. A bare `Mapped[list]` would be read
# as a scalar and hand back None instead of [].
from lembas.db.models.library import KnowledgeBase
ROLE_SYSTEM = "system"
ROLE_USER = "user"
ROLE_ASSISTANT = "assistant"
@@ -83,6 +89,11 @@ class Chat(UUIDPrimaryKey, Timestamps, Base):
cascade="all, delete-orphan",
order_by="Message.created_at",
)
# Which knowledge bases this chat draws on. None means "everything its owner
# can see"; naming some scopes the knowledge tool to those.
knowledge_bases: Mapped[list[KnowledgeBase]] = relationship(
"KnowledgeBase", secondary="chat_knowledge_bases"
)
def __repr__(self) -> str:
return f"<Chat {self.title!r}>"
+62 -2
View File
@@ -23,7 +23,17 @@ hand round, and "share my memories with the team" is a question nobody asked.
from __future__ import annotations
from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy import (
Boolean,
Column,
ForeignKey,
Index,
Integer,
String,
Table,
Text,
UniqueConstraint,
)
from sqlalchemy.orm import Mapped, mapped_column, relationship
from lembas.db.base import Base, Timestamps, UUIDPrimaryKey
@@ -39,13 +49,55 @@ SOURCE_LINK = "link"
# Resource kinds that can be shared. Values are stored, so they are part of the
# schema rather than an implementation detail.
RESOURCE_DOCUMENT = "document"
RESOURCE_BASE = "base"
RESOURCE_NOTE = "note"
RESOURCE_SKILL = "skill"
PRINCIPAL_USER = "user"
PRINCIPAL_GROUP = "group"
# Which knowledge bases a chat draws on. A chat with none searches everything
# its owner can see; a chat with some is scoped to those, which is the point --
# "answer from the contract folder" is a different question from "answer from
# everything I have ever uploaded".
chat_knowledge_bases = Table(
"chat_knowledge_bases",
Base.metadata,
Column("chat_id", String(32), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True),
Column(
"base_id",
String(32),
ForeignKey("knowledge_bases.id", ondelete="CASCADE"),
primary_key=True,
),
)
class KnowledgeBase(UUIDPrimaryKey, Timestamps, Base):
"""A named collection of documents.
Sharing lives here rather than on the individual document: "this folder is
the team's" is the granularity people actually think in, and per-document
grants would mean answering "who can see this?" by checking every file.
A document is visible to whoever can see the base it is in.
"""
__tablename__ = "knowledge_bases"
__table_args__ = (UniqueConstraint("owner_id", "name"),)
owner_id: Mapped[str] = mapped_column(
String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
)
name: Mapped[str] = mapped_column(String(200), nullable=False)
description: Mapped[str] = mapped_column(Text, default="")
documents: Mapped[list[Document]] = relationship(
back_populates="base", cascade="all, delete-orphan"
)
def __repr__(self) -> str:
return f"<KnowledgeBase {self.name!r}>"
class Document(UUIDPrimaryKey, Timestamps, Base):
"""One item in a knowledge library: a file, an image or a saved web page.
@@ -61,6 +113,12 @@ class Document(UUIDPrimaryKey, Timestamps, Base):
owner_id: Mapped[str] = mapped_column(
String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True
)
# Nullable only so the column could be added to an existing table. The
# service always sets it, and a startup sweep files anything that predates
# bases into its owner's default -- see documents.sweep_unfiled.
base_id: Mapped[str | None] = mapped_column(
String(32), ForeignKey("knowledge_bases.id", ondelete="CASCADE"), index=True
)
title: Mapped[str] = mapped_column(String(300), nullable=False)
description: Mapped[str] = mapped_column(Text, default="")
@@ -82,6 +140,8 @@ class Document(UUIDPrimaryKey, Timestamps, Base):
truncated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False)
extraction_error: Mapped[str] = mapped_column(Text, default="")
base: Mapped[KnowledgeBase] = relationship(back_populates="documents")
@property
def is_image(self) -> bool:
return self.kind == "image"
+4
View File
@@ -62,9 +62,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]:
try:
from lembas.db.session import session_scope
from lembas.services.files import sweep_orphans
from lembas.services.library.documents import sweep_unfiled
with session_scope() as db:
sweep_orphans(db)
# Documents that predate knowledge bases have nowhere to live until
# this runs; see services/library/documents.py.
sweep_unfiled(db)
except Exception: # noqa: BLE001 - housekeeping must never block startup
log.exception("orphaned upload sweep failed")
+1 -1
View File
@@ -246,7 +246,7 @@ def build_request(
# behaviour. See services/harness.py for why these are joined rather than
# being two competing layers.
system = harness_service.join(
harness_service.compose(db, user, tools), effective_system_prompt(db, chat)
harness_service.compose(db, user, tools, chat), effective_system_prompt(db, chat)
)
body: dict[str, Any] = {
+1 -1
View File
@@ -175,7 +175,7 @@ async def _run(generation: Generation) -> None:
)
question = _question_from(payload)
needs_title = not chat.title_generated
tool_context = tools_service.context_for(db, owner)
tool_context = tools_service.context_for(db, owner, chat)
for round_number in range(tools_service.MAX_ROUNDS + 1):
accumulator = tools_service.ToolCallAccumulator()
+11
View File
@@ -103,6 +103,7 @@ def compose(
db: DBSession,
user: User | None,
tools: list[dict[str, Any]] | None,
chat=None,
) -> str:
"""The operational preamble for this request, or "" when there is nothing to say."""
families = _families(tools or [])
@@ -129,6 +130,16 @@ def compose(
if block:
parts += ["", "### What you know about this person", "", block]
# Naming the bases a chat is scoped to matters: without it the model cannot
# tell "there is nothing about this" from "I am only allowed to see the
# contracts folder", and phrases a miss as the former.
if "knowledge" in families and chat is not None and chat.knowledge_bases:
names = ", ".join(base.name for base in chat.knowledge_bases)
parts += [
"",
f"Knowledge searches in this chat cover only: {names}.",
]
if "skills" in families:
index = skills_service.index_block(db, user)
if index:
+138 -10
View File
@@ -18,7 +18,7 @@ from sqlalchemy import select
from sqlalchemy.orm import Session as DBSession
from lembas.config import settings
from lembas.db.models import SOURCE_LINK, SOURCE_UPLOAD, Document, User
from lembas.db.models import SOURCE_LINK, SOURCE_UPLOAD, Document, KnowledgeBase, User
from lembas.services import files as files_service
from lembas.services import sharing
from lembas.services.fetch import Fetched
@@ -28,6 +28,10 @@ log = logging.getLogger(__name__)
INDEX = "documents_fts"
# What a first base is called when one has to be invented -- on the first
# upload, or for documents that predate bases existing.
DEFAULT_BASE_NAME = "My documents"
# How much of a document's text a search result carries back to the model. A
# whole 100-page extract would swallow the context window; this is enough to
# judge relevance and to answer from, and `knowledge_get` fetches the rest.
@@ -57,12 +61,109 @@ def stored_path(stored_name: str) -> Path | None:
return path if path.is_file() else None
# --- Bases -------------------------------------------------------------------
def visible_bases(db: DBSession, user: User | None):
return select(KnowledgeBase).where(sharing.visible_to(KnowledgeBase, user))
def get_base(db: DBSession, base_id: str, user: User | None) -> KnowledgeBase | None:
base = db.get(KnowledgeBase, base_id)
if base is None or not sharing.can_read(db, base, user):
return None
return base
def create_base(
db: DBSession, *, owner: User, name: str, description: str = ""
) -> KnowledgeBase:
name = " ".join((name or "").split())[:200] or DEFAULT_BASE_NAME
existing = db.scalar(
select(KnowledgeBase).where(
KnowledgeBase.owner_id == owner.id, KnowledgeBase.name == name
)
)
if existing is not None:
raise ValueError(f"You already have a knowledge base called {name!r}.")
base = KnowledgeBase(owner_id=owner.id, name=name, description=description.strip()[:2000])
db.add(base)
db.commit()
return base
def default_base(db: DBSession, owner: User) -> KnowledgeBase:
"""The base a document goes into when none was chosen.
Made on demand rather than at registration, so an account that never uses
the library never grows an empty one.
"""
base = db.scalar(
select(KnowledgeBase)
.where(KnowledgeBase.owner_id == owner.id)
.order_by(KnowledgeBase.created_at)
)
if base is not None:
return base
base = KnowledgeBase(owner_id=owner.id, name=DEFAULT_BASE_NAME)
db.add(base)
db.commit()
return base
def delete_base(db: DBSession, base: KnowledgeBase) -> None:
"""Delete a base and everything in it.
The documents go too -- a base is a place, not a label, and leaving its
contents behind with nowhere to live would need an "unfiled" concept that
exists only to hold the wreckage of deletes.
"""
for document in list(base.documents):
path = stored_path(document.stored_name)
if path is not None:
path.unlink(missing_ok=True)
sharing.forget_resource(db, base)
db.delete(base)
db.commit()
def sweep_unfiled(db: DBSession) -> int:
"""File documents that predate knowledge bases into their owner's default.
`Document.base_id` is nullable only because the column had to be added to a
table that already had rows. This is what makes "always set" true in
practice, and it runs at startup beside the orphaned-upload sweep.
"""
unfiled = list(db.scalars(select(Document).where(Document.base_id.is_(None))))
if not unfiled:
return 0
bases: dict[str, KnowledgeBase] = {}
for document in unfiled:
owner = db.get(User, document.owner_id)
if owner is None:
continue
if owner.id not in bases:
bases[owner.id] = default_base(db, owner)
document.base_id = bases[owner.id].id
db.commit()
log.info("filed %d document(s) that predated knowledge bases", len(unfiled))
return len(unfiled)
# --- Creating ----------------------------------------------------------------
def store_upload(
db: DBSession, *, owner: User, payload: bytes, filename: str, title: str = ""
db: DBSession,
*,
owner: User,
payload: bytes,
filename: str,
title: str = "",
base: KnowledgeBase | None = None,
) -> Document:
"""Add an uploaded file to the library. Raises files.FileError if unusable."""
prepared = files_service.prepare(payload, filename)
base = base or default_base(db, owner)
stored_name = f"{secrets.token_hex(16)}{prepared.extension}"
(library_dir() / stored_name).write_bytes(prepared.payload)
@@ -70,6 +171,7 @@ def store_upload(
display = files_service.safe_display_name(filename)
document = Document(
owner_id=owner.id,
base_id=base.id,
title=(title.strip() or display)[:300],
source=SOURCE_UPLOAD,
filename=display,
@@ -90,14 +192,18 @@ def store_upload(
return document
def store_page(db: DBSession, *, owner: User, page: Fetched) -> Document:
def store_page(
db: DBSession, *, owner: User, page: Fetched, base: KnowledgeBase | None = None
) -> Document:
"""Add a fetched web page to the library.
Saved as text rather than as the original HTML: the point of keeping it is
what it said, and the markup would have to be reduced again on every read.
"""
base = base or default_base(db, owner)
document = Document(
owner_id=owner.id,
base_id=base.id,
title=page.title[:300] or page.url[:300],
source=SOURCE_LINK,
source_url=page.url,
@@ -115,19 +221,41 @@ def store_page(db: DBSession, *, owner: User, page: Fetched) -> Document:
# --- Reading -----------------------------------------------------------------
def visible(db: DBSession, user: User | None):
return select(Document).where(sharing.visible_to(Document, user))
def visible(db: DBSession, user: User | None, *, base_ids: list[str] | None = None):
"""Documents this user may see, optionally narrowed to some bases.
Visibility comes from the base, not the document: a document is readable by
whoever can read the base it lives in. That is the whole reason bases are
shareable and documents are not.
"""
condition = Document.base_id.in_(
select(KnowledgeBase.id).where(sharing.visible_to(KnowledgeBase, user))
)
query = select(Document).where(condition)
if base_ids:
# Still filtered by visibility above, so naming a base you cannot see
# returns nothing rather than granting access to it.
query = query.where(Document.base_id.in_(base_ids))
return query
def get(db: DBSession, document_id: str, user: User | None) -> Document | None:
document = db.get(Document, document_id)
if document is None or not sharing.can_read(db, document, user):
if document is None:
return None
base = db.get(KnowledgeBase, document.base_id) if document.base_id else None
if base is None or not sharing.can_read(db, base, user):
return None
return document
def search(
db: DBSession, user: User | None, needle: str, *, limit: int = 10
db: DBSession,
user: User | None,
needle: str,
*,
limit: int = 10,
base_ids: list[str] | None = None,
) -> list[Document]:
"""Documents matching `needle` that this user may see, best match first.
@@ -141,7 +269,9 @@ def search(
order = {hit.id: position for position, hit in enumerate(hits)}
rows = list(
db.scalars(visible(db, user).where(Document.id.in_(list(order))))
db.scalars(
visible(db, user, base_ids=base_ids).where(Document.id.in_(list(order)))
)
)
rows.sort(key=lambda document: order.get(document.id, len(order)))
return rows[:limit]
@@ -160,7 +290,5 @@ def delete(db: DBSession, document: Document) -> None:
path = stored_path(document.stored_name)
if path is not None:
path.unlink(missing_ok=True)
# Shares carry no foreign key to their resource, so nothing cascades.
sharing.forget_resource(db, document)
db.delete(document)
db.commit()
+8 -3
View File
@@ -3,6 +3,11 @@
One rule, in one place, for all three: you can see a resource if you own it, if
it was shared with you by name, or if it was shared with a group you are in.
Documents are deliberately absent from that list. They are shared through the
knowledge base they belong to -- "this folder is the team's" is the granularity
people think in, and per-document grants would mean answering "who can see
this?" by checking every file. See services.library.documents.visible.
Everything that lists or searches a library store goes through `visible_to`.
Writing the same condition into each query would work right up until one of
them was written slightly differently, and the way that failure shows up is
@@ -26,10 +31,10 @@ from sqlalchemy.orm import Session as DBSession
from lembas.db.models import (
PRINCIPAL_GROUP,
PRINCIPAL_USER,
RESOURCE_DOCUMENT,
RESOURCE_BASE,
RESOURCE_NOTE,
RESOURCE_SKILL,
Document,
KnowledgeBase,
Note,
Share,
Skill,
@@ -41,7 +46,7 @@ log = logging.getLogger(__name__)
# The mapping between a model class and the string stored in Share. Kept here
# so no caller has to remember which literal goes with which table.
RESOURCE_TYPES: dict[Any, str] = {
Document: RESOURCE_DOCUMENT,
KnowledgeBase: RESOURCE_BASE,
Note: RESOURCE_NOTE,
Skill: RESOURCE_SKILL,
}
+8 -2
View File
@@ -73,6 +73,9 @@ class ToolContext:
owner_id: str
search_config: dict[str, Any] = field(default_factory=dict)
allow_private_fetch: bool = False
# Which knowledge bases this chat is scoped to. Empty means "everything the
# owner can see", which is what a chat with none attached should do.
base_ids: list[str] = field(default_factory=list)
@dataclass
@@ -171,7 +174,9 @@ async def _run_knowledge_search(context: ToolContext, args: dict[str, Any]) -> T
with session_scope() as db:
user = db.get(User, context.owner_id)
found = documents_service.search(db, user, query, limit=6)
found = documents_service.search(
db, user, query, limit=6, base_ids=context.base_ids
)
event = {
"name": "knowledge_search",
"query": query,
@@ -677,11 +682,12 @@ def enabled_tools(db: DBSession, chat: Chat, user: User | None) -> list[dict[str
return [tool.schema for tool in REGISTRY.values() if tool.family in families]
def context_for(db: DBSession, user: User | None) -> ToolContext:
def context_for(db: DBSession, user: User | None, chat: Chat | None = None) -> ToolContext:
"""The snapshot a running tool needs, taken while the session is open."""
return ToolContext(
owner_id=user.id if user else "",
search_config=settings_store.search(db),
base_ids=[base.id for base in chat.knowledge_bases] if chat is not None else [],
)
+37
View File
@@ -231,6 +231,43 @@ button, input, textarea, select {
box-shadow: 0 0 0 3px var(--accent-soft);
}
.input::placeholder, .textarea::placeholder { color: var(--ink-faint); }
/*
File inputs.
A file input is two things in one box -- a button the browser draws and the
chosen filename beside it -- and neither inherits anything useful. Left alone
with `.input`, the padding applies to the whole control so the button sits
hard against the left edge while the text floats off its centre line.
So: no horizontal padding on the control, the button styled to the same
height as everything else and given the right border that separates it, and
the filename centred with line-height rather than flexbox, which file inputs
do not lay out reliably.
*/
.input[type="file"] {
padding: 0 var(--control-px) 0 0;
line-height: calc(var(--control-h) - 2px);
cursor: pointer;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
color: var(--ink-muted);
}
.input[type="file"]::file-selector-button {
height: calc(var(--control-h) - 2px);
margin: 0 var(--sp-3) 0 0;
padding: 0 var(--control-px);
border: 0;
border-right: 1px solid var(--border);
background: var(--surface-hover);
color: var(--ink);
font: inherit;
font-weight: 500;
cursor: pointer;
transition: background var(--transition-fast);
}
.input[type="file"]:hover::file-selector-button { background: var(--surface-active); }
.input--mono, .textarea--mono { font-family: var(--font-mono); font-size: var(--text-xs); }
.select {
@@ -25,7 +25,7 @@
<span class="picker__option-body">
<span class="picker__option-name">{{ document.title }}</span>
<span class="picker__option-note">
{{ document.kind }}
{% if document.base %}{{ document.base.name }} · {% endif %}{{ document.kind }}
{%- if document.pages %} · {{ document.pages }}p{% endif %}
{%- if document.owner_id != user.id %} · shared{% endif %}
</span>
+30
View File
@@ -68,6 +68,36 @@
</div>
{% endif %}
{% if knowledge_bases %}
{# Which bases this chat draws on. None ticked means everything you can
see, which is what a chat with nothing chosen should do. #}
<div class="field">
<label class="field__label">Knowledge</label>
<form hx-patch="/api/chats/{{ chat.id }}" hx-swap="none" hx-trigger="change">
{# Always submitted, so unticking the last box still says something.
An absent checkbox carries no signal of its own. #}
<input type="hidden" name="knowledge_base_ids" value="">
<div class="checkbox-row">
{% for base in knowledge_bases %}
<label class="checkbox">
<input type="checkbox" name="knowledge_base_ids" value="{{ base.id }}"
{{ 'checked' if base.id in attached_base_ids }}>
<span>{{ base.name }}</span>
</label>
{% endfor %}
</div>
</form>
<p class="field__hint">
{% if attached_base_ids %}
Searches in this chat are limited to what is ticked.
{% else %}
Nothing ticked, so this chat can search everything in your
<a href="/library/knowledge">library</a>.
{% endif %}
</p>
</div>
{% endif %}
{% if can.get("chat.params") %}
<div class="grid grid--3">
<div class="field">
@@ -0,0 +1,126 @@
{% extends "library/_layout.html" %}
{% from "_macros.html" import icon %}
{% set section = "knowledge" %}
{% block title %}{{ base.name }} - LLeMbas{% endblock %}
{% block heading %}{{ base.name }}{% endblock %}
{% block library_content %}
<div class="btn-row" style="margin-bottom: var(--sp-5)">
<a class="btn btn--sm" href="/library/knowledge">
{{ icon("chevron-right", "icon--sm") }} All knowledge bases
</a>
</div>
{% if is_owner %}
<section class="card">
<h2 class="card__title">Add to this base</h2>
<div class="grid grid--2">
<form method="post" action="/api/library/documents" enctype="multipart/form-data">
<input type="hidden" name="base_id" value="{{ base.id }}">
<div class="field">
<label class="field__label" for="doc-file">A file</label>
<input class="input" id="doc-file" type="file" name="file" required
accept="image/*,.pdf,.txt,.md,.csv,.json,.py,.js,.ts,.rs,.go,.sh,.sql,.yaml,.yml,.toml,.log">
<p class="field__hint">
Images, PDFs and text. A PDF has its text read once, now.
</p>
</div>
<button class="btn btn--primary" type="submit">Upload</button>
</form>
<form method="post" action="/api/library/documents/link">
<input type="hidden" name="base_id" value="{{ base.id }}">
<div class="field">
<label class="field__label" for="doc-url">A web page</label>
<input class="input" id="doc-url" type="url" name="url" required
placeholder="https://example.com/article">
<p class="field__hint">
Fetched now and kept as text, so it survives the page changing.
</p>
</div>
<button class="btn" type="submit">Save page</button>
</form>
</div>
</section>
{% endif %}
<form method="get" action="/library/knowledge/{{ base.id }}" class="btn-row"
style="margin-bottom: var(--sp-5)">
<input class="input" type="search" name="q" value="{{ q }}" style="flex: 1"
placeholder="Search this base…">
<button class="btn" type="submit">{{ icon("search", "icon--sm") }} Search</button>
{% if q %}<a class="btn btn--sm" href="/library/knowledge/{{ base.id }}">Clear</a>{% endif %}
</form>
{% if not documents %}
<div class="empty">
{{ icon("archive", "empty__mark") }}
<h2 class="empty__title">{{ "Nothing found" if q else "Empty" }}</h2>
<p class="empty__text">
{% if q %}Nothing in this base matches “{{ q }}”.
{% else %}Add a file or a web page above.{% endif %}
</p>
</div>
{% else %}
<ul class="model-list">
{% for document in documents %}
<li class="model-list__item">
<div style="min-width: 0">
<a href="/library/knowledge/document/{{ document.id }}">
<strong>{{ document.title }}</strong>
</a>
<div class="text-xs faint">
{{ document.kind }}
{%- if document.pages %} · {{ document.pages }} page{{ '' if document.pages == 1 else 's' }}{% endif %}
{%- if document.size_bytes %} · {{ document.human_size }}{% endif %}
{%- if document.source_url %} · {{ document.source_url[:60] }}{% endif %}
</div>
{% if document.description %}
<div class="text-xs faint">{{ document.description }}</div>
{% endif %}
</div>
<div class="btn-row">
{% if document.extraction_error %}
<span class="badge badge--danger" title="{{ document.extraction_error }}">no text</span>
{% endif %}
</div>
</li>
{% endfor %}
</ul>
{% include "library/_pager.html" %}
{% endif %}
<form method="post" action="/api/library/bases/{{ base.id }}" style="margin-top: var(--sp-8)">
<section class="card">
<h2 class="card__title">This base</h2>
<div class="grid grid--2">
<div class="field">
<label class="field__label" for="name">Name</label>
<input class="input" id="name" name="name" value="{{ base.name }}" maxlength="200"
{{ 'disabled' if not is_owner }}>
</div>
<div class="field">
<label class="field__label" for="description">Description</label>
<input class="input" id="description" name="description" maxlength="2000"
value="{{ base.description }}" {{ 'disabled' if not is_owner }}>
</div>
</div>
</section>
{% include "library/_share.html" %}
{% if is_owner %}
<div class="form-actions">
<button class="btn btn--primary" type="submit">Save</button>
<span class="spacer"></span>
<button class="btn btn--danger" type="submit"
formaction="/api/library/bases/{{ base.id }}/delete"
data-confirm-button="Delete “{{ base.name }}” and the {{ pager.total }} document(s) in it? Messages they were attached to keep their copies."
data-confirm-title="Delete knowledge base">
{{ icon("trash", "icon--sm") }} Delete base
</button>
</div>
{% endif %}
</form>
{% endblock %}
+41 -66
View File
@@ -7,85 +7,60 @@
{% block library_content %}
<p class="admin-lede">
Documents, images and saved web pages you have collected. A model with the
knowledge tool searches these before it searches the web, and you can attach
any of them to a message.
Knowledge bases are collections of documents, images and saved web pages. Keep
them separate — one per subject, project or client — and a chat can be pointed
at just the ones it should draw on. Sharing happens here too: share a base and
everything in it comes with it.
</p>
<section class="card">
<h2 class="card__title">Add</h2>
<div class="grid grid--2">
<form method="post" action="/api/library/documents" enctype="multipart/form-data">
<div class="field">
<label class="field__label" for="doc-file">A file</label>
<input class="input" id="doc-file" type="file" name="file" required
accept="image/*,.pdf,.txt,.md,.csv,.json,.py,.js,.ts,.rs,.go,.sh,.sql,.yaml,.yml,.toml,.log">
<p class="field__hint">
Images, PDFs and text. PDFs have their text read once, now.
</p>
</div>
<button class="btn btn--primary" type="submit">Upload</button>
</form>
{% if error %}
<div class="alert alert--error">{{ icon("warning", "alert__icon") }} <span>{{ error }}</span></div>
{% endif %}
<form method="post" action="/api/library/documents/link">
<div class="field">
<label class="field__label" for="doc-url">A web page</label>
<input class="input" id="doc-url" type="url" name="url" required
placeholder="https://example.com/article">
<p class="field__hint">
Fetched now and kept as text, so it survives the page changing.
</p>
</div>
<button class="btn" type="submit">Save page</button>
</form>
</div>
</section>
<form method="get" action="/library/knowledge" class="btn-row" style="margin-bottom: var(--sp-5)">
<input class="input" type="search" name="q" value="{{ q }}" style="flex: 1"
placeholder="Search titles and contents…">
<button class="btn" type="submit">{{ icon("search", "icon--sm") }} Search</button>
{% if q %}<a class="btn btn--sm" href="/library/knowledge">Clear</a>{% endif %}
</form>
{% if not documents %}
<div class="empty">
{{ icon("archive", "empty__mark") }}
<h2 class="empty__title">{{ "Nothing found" if q else "The shelves are bare" }}</h2>
<p class="empty__text">
{% if q %}
No document matches “{{ q }}”.
{% else %}
Add a file or a web page above and it becomes searchable — by you, and by
any model you have given the knowledge tool.
{% endif %}
</p>
</div>
{% else %}
{% if bases %}
<ul class="model-list">
{% for document in documents %}
{% for base in bases %}
<li class="model-list__item">
<div style="min-width: 0">
<a href="/library/knowledge/{{ document.id }}"><strong>{{ document.title }}</strong></a>
<a href="/library/knowledge/{{ base.id }}"><strong>{{ base.name }}</strong></a>
<div class="text-xs faint">
{{ document.kind }}
{%- if document.pages %} · {{ document.pages }} page{{ '' if document.pages == 1 else 's' }}{% endif %}
{%- if document.size_bytes %} · {{ document.human_size }}{% endif %}
{%- if document.source_url %} · {{ document.source_url[:60] }}{% endif %}
{{ counts.get(base.id, 0) }} document{{ '' if counts.get(base.id, 0) == 1 else 's' }}
{%- if base.description %} · {{ base.description }}{% endif %}
</div>
{% if document.description %}
<div class="text-xs faint">{{ document.description }}</div>
{% endif %}
</div>
<div class="btn-row">
{% if document.extraction_error %}
<span class="badge badge--danger" title="{{ document.extraction_error }}">no text</span>
{% endif %}
{% if document.owner_id != user.id %}<span class="badge">shared</span>{% endif %}
{% if base.owner_id != user.id %}<span class="badge">shared with you</span>{% endif %}
</div>
</li>
{% endfor %}
</ul>
{% include "library/_pager.html" %}
{% else %}
<div class="empty">
{{ icon("archive", "empty__mark") }}
<h2 class="empty__title">No knowledge bases yet</h2>
<p class="empty__text">
Make one below, then put documents in it. A chat with no base attached
searches everything you have; a chat pointed at one searches only that.
</p>
</div>
{% endif %}
<section class="card" style="margin-top: var(--sp-6)">
<h2 class="card__title">New knowledge base</h2>
<form method="post" action="/api/library/bases">
<div class="grid grid--2">
<div class="field">
<label class="field__label" for="base-name">Name</label>
<input class="input" id="base-name" name="name" required maxlength="200"
placeholder="Contracts">
</div>
<div class="field">
<label class="field__label" for="base-description">Description</label>
<input class="input" id="base-description" name="description" maxlength="2000"
placeholder="What belongs in here.">
</div>
</div>
<button class="btn btn--primary" type="submit">{{ icon("plus", "icon--sm") }} Create</button>
</form>
</section>
{% endblock %}
@@ -7,7 +7,9 @@
{% block library_content %}
<div class="btn-row" style="margin-bottom: var(--sp-5)">
<a class="btn btn--sm" href="/library/knowledge">{{ icon("chevron-right", "icon--sm") }} All documents</a>
<a class="btn btn--sm" href="/library/knowledge/{{ document.base_id }}">
{{ icon("chevron-right", "icon--sm") }} Back to {{ document.base.name if document.base else "the base" }}
</a>
</div>
<form method="post" action="/api/library/documents/{{ document.id }}">
@@ -28,6 +30,22 @@
</p>
</div>
{% if document.base %}
<div class="field">
<label class="field__label" for="base_id">Knowledge base</label>
{# Moving a document changes who can see it, which is the point of bases.
Only bases this person can write to are offered. #}
<select class="select" id="base_id" name="base_id" {{ 'disabled' if not is_owner }}>
{% for option in user_bases %}
<option value="{{ option.id }}" {{ 'selected' if option.id == document.base_id }}>
{{ option.name }}
</option>
{% endfor %}
</select>
<p class="field__hint">Moving it changes who can see it.</p>
</div>
{% endif %}
<dl class="detail-list">
<dt>Kind</dt><dd>{{ document.kind }}</dd>
{% if document.source_url %}
@@ -50,8 +68,6 @@
{% endif %}
</section>
{% include "library/_share.html" %}
{% if is_owner %}
<div class="form-actions">
<button class="btn btn--primary" type="submit">Save</button>