From 35b9d8c8d2262f5137a1950269e9529980b55538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jaroslav=20Bene=C5=A1?= Date: Tue, 21 Jul 2026 20:00:15 +0200 Subject: [PATCH] Knowledge bases, and a file input that lines up MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit **Bases.** Documents now live in named collections rather than one flat pile, and a chat can be pointed at particular ones — "answer from the contracts folder" is a different question from "answer from everything I have ever uploaded". A chat with none attached still searches everything its owner can see, because empty means unscoped, not empty. The harness names the attached bases. Without that the model cannot tell "there is nothing about this" from "I am only allowed to see one folder", and it phrases a miss as the former. **Sharing moves to the base.** A document is visible to whoever can see the base it lives in, so `Document` is gone from the shareable types and `documents.visible()` filters through `base_id`. "This folder is the team's" is the granularity people think in; per-document grants meant answering "who can see this?" by checking every file. Moving a document between bases changes who can see it, so the destination has to be one you own. `Document.base_id` is nullable only because the column had to be added to a table that already had rows. `sweep_unfiled()` runs at startup beside the orphaned-upload sweep and files anything predating bases into its owner's default, which is what makes "always set" true everywhere else. **The file input.** `.input` gave it a fixed height and horizontal padding, so the browser's own button sat hard against the left edge while the filename floated off the centre line. A file input is two controls in one box and neither inherits anything useful, so it gets its own rule: no horizontal padding, the button sized to `--control-h` with the divider that separates it, and the text centred with line-height rather than flexbox, which file inputs do not lay out reliably. 437 tests, ruff clean. Co-Authored-By: Claude Opus 4.8 (1M context) --- README.md | 8 +- src/lembas/api/chats.py | 20 ++ src/lembas/api/library.py | 174 +++++++++++++++--- src/lembas/api/pages.py | 18 +- src/lembas/db/models/__init__.py | 8 +- src/lembas/db/models/chat.py | 13 +- src/lembas/db/models/library.py | 64 ++++++- src/lembas/main.py | 4 + src/lembas/services/chat.py | 2 +- src/lembas/services/generation.py | 2 +- src/lembas/services/harness.py | 11 ++ src/lembas/services/library/documents.py | 148 ++++++++++++++- src/lembas/services/sharing.py | 11 +- src/lembas/services/tools.py | 10 +- src/lembas/web/static/css/app.css | 37 ++++ .../web/templates/chat/_knowledge_picker.html | 2 +- src/lembas/web/templates/chat/index.html | 30 +++ .../web/templates/library/base_detail.html | 126 +++++++++++++ .../web/templates/library/knowledge.html | 107 +++++------ .../templates/library/knowledge_detail.html | 22 ++- tests/test_sharing.py | 87 +++++++-- tests/test_tools.py | 42 +++++ 22 files changed, 809 insertions(+), 137 deletions(-) create mode 100644 src/lembas/web/templates/library/base_detail.html diff --git a/README.md b/README.md index af465fd..7e274d7 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,8 @@ runtime. Clone it, `pip install -e .`, run it. any OpenAI-compatible audio endpoint (whisper.cpp, Speaches, Kokoro…). Each person picks their own voice - **A library** — four places a model can reach for. **Knowledge**: documents, - images and web pages you collect, searched before the web. **Notes**: longer + images and web pages you collect, grouped into named bases so a chat can be + pointed at just the right one, searched before the web. **Notes**: longer things it writes down and finds again later. **Memory**: short facts about you, in front of it on every turn. **Skills**: saved procedures it can follow, and write. All of it visible and editable by you, and shareable with a group or a @@ -143,6 +144,11 @@ model: give it the tools it should have under **Admin → Models**, where `tools` decides whether a tool list may be sent at all and the built-in tools are chosen one by one. +Knowledge is organised into **bases** — one per subject, project or client. A +chat with no base attached searches everything you have; tick some in the chat's +settings panel and it searches only those. Sharing happens at the base: share it +and everything in it comes too, read-only. + Search is SQLite's FTS5 — keyword matching with BM25 ranking, no embedding service to run and nothing that stops working offline. It will not match a paraphrase, so a line of description on a document is worth writing. diff --git a/src/lembas/api/chats.py b/src/lembas/api/chats.py index d3d7dde..59c127b 100644 --- a/src/lembas/api/chats.py +++ b/src/lembas/api/chats.py @@ -484,6 +484,26 @@ async def update_chat(request: Request, db: Db, user: RequiredUser, chat_id: str ) chat.system_prompt = str(form["system_prompt"]).strip()[:8000] + if "knowledge_base_ids" in form: + # Sent as a single field even when empty, so that clearing every box + # actually clears the attachment -- absent checkboxes carry no signal of + # their own, which is the same trap update_chat exists to avoid. + from lembas.db.models import KnowledgeBase + from lembas.services.library import documents as documents_service + + wanted = [value for value in form.getlist("knowledge_base_ids") if value] + chat.knowledge_bases = ( + list( + db.scalars( + documents_service.visible_bases(db, user).where( + KnowledgeBase.id.in_(wanted) + ) + ) + ) + if wanted + else [] + ) + submitted_params = {name: form[name] for name in _PARAM_RANGES if name in form} if submitted_params: if not allowed.get("chat.params"): diff --git a/src/lembas/api/library.py b/src/lembas/api/library.py index e8f37fd..4a85d7f 100644 --- a/src/lembas/api/library.py +++ b/src/lembas/api/library.py @@ -27,6 +27,7 @@ from lembas.db.models import ( PRINCIPAL_USER, Document, Group, + KnowledgeBase, Note, Skill, SkillRevision, @@ -92,33 +93,54 @@ async def library_home(user: RequiredUser): # --- Knowledge --------------------------------------------------------------- +# Route order matters: /library/knowledge/document/{id} must be registered +# before /library/knowledge/{base_id}, or "document" is parsed as a base id. +# FastAPI matches in registration order and this has bitten before. @router.get("/library/knowledge") -async def knowledge_list( - request: Request, db: Db, user: RequiredUser, q: str = "", page: int = 1, saved: str = "" -): - if q.strip(): - # Search returns best-match order and its own limit, so it is not paged. - rows = documents_service.search(db, user, q, limit=PAGE_SIZE) - pager = {"page": 1, "pages": 1, "total": len(rows)} - else: - rows, pager = _page( - db, documents_service.visible(db, user).order_by(Document.created_at.desc()), page +async def knowledge_list(request: Request, db: Db, user: RequiredUser, error: str = ""): + """The bases, not the documents. A library is a set of places first.""" + bases = list(db.scalars(documents_service.visible_bases(db, user).order_by(KnowledgeBase.name))) + counts = { + base.id: db.scalar( + select(func.count()).select_from(Document).where(Document.base_id == base.id) ) + or 0 + for base in bases + } return render( request, "library/knowledge.html", { "section": "knowledge", - "documents": rows, - "q": q, - "pager": pager, - "saved": saved, + "bases": bases, + "counts": counts, + "error": error, **sidebar_context(db, user), }, ) -@router.get("/library/knowledge/{document_id}") +@router.post("/api/library/bases") +async def create_base( + db: Db, user: RequiredUser, name: str = Form(""), description: str = Form("") +) -> Response: + try: + base = documents_service.create_base( + db, owner=user, name=name, description=description + ) + except ValueError as exc: + from urllib.parse import quote + + return RedirectResponse( + f"/library/knowledge?error={quote(str(exc))}", + status_code=status.HTTP_303_SEE_OTHER, + ) + return RedirectResponse( + f"/library/knowledge/{base.id}", status_code=status.HTTP_303_SEE_OTHER + ) + + +@router.get("/library/knowledge/document/{document_id}") async def knowledge_detail(request: Request, db: Db, user: RequiredUser, document_id: str): document = documents_service.get(db, document_id, user) if document is None: @@ -129,38 +151,129 @@ async def knowledge_detail(request: Request, db: Db, user: RequiredUser, documen { "section": "knowledge", "document": document, - **_shared_context(db, user, document), + "is_owner": sharing.can_write(document, user), + # Only bases this person owns: moving a document into one they can + # merely read would hand it to that base's owner. + "user_bases": list( + db.scalars( + select(KnowledgeBase) + .where(KnowledgeBase.owner_id == user.id) + .order_by(KnowledgeBase.name) + ) + ), **sidebar_context(db, user), }, ) +@router.get("/library/knowledge/{base_id}") +async def base_detail( + request: Request, db: Db, user: RequiredUser, base_id: str, q: str = "", page: int = 1 +): + base = documents_service.get_base(db, base_id, user) + if base is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That knowledge base is not available.") + + if q.strip(): + rows = documents_service.search(db, user, q, limit=PAGE_SIZE, base_ids=[base.id]) + pager = {"page": 1, "pages": 1, "total": len(rows)} + else: + rows, pager = _page( + db, + documents_service.visible(db, user, base_ids=[base.id]).order_by( + Document.created_at.desc() + ), + page, + ) + return render( + request, + "library/base_detail.html", + { + "section": "knowledge", + "base": base, + "documents": rows, + "q": q, + "pager": pager, + **_shared_context(db, user, base), + **sidebar_context(db, user), + }, + ) + + +@router.post("/api/library/bases/{base_id}") +async def update_base(request: Request, db: Db, user: RequiredUser, base_id: str) -> Response: + base = documents_service.get_base(db, base_id, user) + if base is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That knowledge base is not available.") + if not sharing.can_write(base, user): + raise HTTPException(status.HTTP_403_FORBIDDEN, "That base is not yours to change.") + + form = await request.form() + name = " ".join(str(form.get("name", "")).split())[:200] + if name: + base.name = name + base.description = str(form.get("description", "")).strip()[:2000] + db.commit() + _apply_shares(db, user, base, form) + return RedirectResponse( + f"/library/knowledge/{base.id}", status_code=status.HTTP_303_SEE_OTHER + ) + + +@router.post("/api/library/bases/{base_id}/delete") +async def delete_base(db: Db, user: RequiredUser, base_id: str) -> Response: + base = documents_service.get_base(db, base_id, user) + if base is None or not sharing.can_write(base, user): + raise HTTPException(status.HTTP_404_NOT_FOUND, "That knowledge base is not available.") + documents_service.delete_base(db, base) + return RedirectResponse("/library/knowledge", status_code=status.HTTP_303_SEE_OTHER) + + @router.post("/api/library/documents") async def upload_document( - db: Db, user: RequiredUser, file: UploadFile = File(...), title: str = Form("") + db: Db, + user: RequiredUser, + file: UploadFile = File(...), + title: str = Form(""), + base_id: str = Form(""), ) -> Response: + base = documents_service.get_base(db, base_id, user) if base_id else None + if base is not None and not sharing.can_write(base, user): + raise HTTPException(status.HTTP_403_FORBIDDEN, "That base is not yours to add to.") + payload = await file.read(files_service.MAX_UPLOAD_BYTES + 1) try: document = documents_service.store_upload( - db, owner=user, payload=payload, filename=file.filename or "file", title=title + db, + owner=user, + payload=payload, + filename=file.filename or "file", + title=title, + base=base, ) except files_service.FileError as exc: raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc return RedirectResponse( - f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER + f"/library/knowledge/{document.base_id}", status_code=status.HTTP_303_SEE_OTHER ) @router.post("/api/library/documents/link") -async def save_link(db: Db, user: RequiredUser, url: str = Form(...)) -> Response: +async def save_link( + db: Db, user: RequiredUser, url: str = Form(...), base_id: str = Form("") +) -> Response: + base = documents_service.get_base(db, base_id, user) if base_id else None + if base is not None and not sharing.can_write(base, user): + raise HTTPException(status.HTTP_403_FORBIDDEN, "That base is not yours to add to.") + config = settings_store.search(db) try: page = await fetch(url, allow_private=bool(config.get("allow_private_fetch"))) except FetchError as exc: raise HTTPException(status.HTTP_400_BAD_REQUEST, exc.message) from exc - document = documents_service.store_page(db, owner=user, page=page) + document = documents_service.store_page(db, owner=user, page=page, base=base) return RedirectResponse( - f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER + f"/library/knowledge/{document.base_id}", status_code=status.HTTP_303_SEE_OTHER ) @@ -177,10 +290,18 @@ async def update_document( form = await request.form() document.title = str(form.get("title", document.title)).strip()[:300] or document.title document.description = str(form.get("description", "")).strip()[:2000] + + # Moving between bases changes who can see it, which is the whole point of + # bases -- so the destination has to be one this person can write to. + wanted = str(form.get("base_id", "")).strip() + if wanted and wanted != document.base_id: + destination = documents_service.get_base(db, wanted, user) + if destination is not None and sharing.can_write(destination, user): + document.base_id = destination.id + db.commit() - _apply_shares(db, user, document, form) return RedirectResponse( - f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER + f"/library/knowledge/document/{document.id}", status_code=status.HTTP_303_SEE_OTHER ) @@ -189,8 +310,11 @@ async def delete_document(db: Db, user: RequiredUser, document_id: str) -> Respo document = documents_service.get(db, document_id, user) if document is None or not sharing.can_write(document, user): raise HTTPException(status.HTTP_404_NOT_FOUND, "That document is not available.") + base_id = document.base_id documents_service.delete(db, document) - return RedirectResponse("/library/knowledge", status_code=status.HTTP_303_SEE_OTHER) + return RedirectResponse( + f"/library/knowledge/{base_id}", status_code=status.HTTP_303_SEE_OTHER + ) @router.get("/api/library/documents/{document_id}/content") diff --git a/src/lembas/api/pages.py b/src/lembas/api/pages.py index 7ec32a0..79a79d2 100644 --- a/src/lembas/api/pages.py +++ b/src/lembas/api/pages.py @@ -8,11 +8,12 @@ from sqlalchemy import select from sqlalchemy.orm import Session as DBSession from lembas.api.deps import Db, RequiredUser -from lembas.db.models import Chat, Folder, Message, User +from lembas.db.models import Chat, Folder, KnowledgeBase, Message, User from lembas.security import permissions from lembas.services import audio as audio_service from lembas.services import chat as chat_service from lembas.services import settings_store +from lembas.services.library import documents as documents_service from lembas.services.markdown import render_markdown from lembas.web.templating import STATIC_DIR, render @@ -42,6 +43,19 @@ def _chat_context(db: DBSession, user: User, chat: Chat | None) -> dict: # may not be the model the chat is set to now. Keyed by model_id, the # denormalised value stored on each message. "models_by_id": {m.model_id: m for m in models}, + # Offered in the chat settings panel so a conversation can be pointed at + # particular bases. Empty when the reader has none, and the panel then + # shows nothing rather than an empty control. + "knowledge_bases": ( + list( + db.scalars( + documents_service.visible_bases(db, user).order_by(KnowledgeBase.name) + ) + ) + if permissions.has(db, user, "library.use") + else [] + ), + "attached_base_ids": [base.id for base in chat.knowledge_bases] if chat else [], **audio_service.template_flags(db, user), } @@ -223,8 +237,6 @@ async def chat_detail(request: Request, db: Db, user: RequiredUser, chat_id: str if current is not None and (current.system_prompt or "").strip(): inherited, inherited_from = current.system_prompt.strip(), "model" else: - from lembas.services import settings_store - instance_prompt = (settings_store.get(db, "system_prompt") or "").strip() if instance_prompt: inherited, inherited_from = instance_prompt, "instance" diff --git a/src/lembas/db/models/__init__.py b/src/lembas/db/models/__init__.py index c6edaeb..8634c11 100644 --- a/src/lembas/db/models/__init__.py +++ b/src/lembas/db/models/__init__.py @@ -26,17 +26,19 @@ from lembas.db.models.library import ( AUTHOR_USER, PRINCIPAL_GROUP, PRINCIPAL_USER, - RESOURCE_DOCUMENT, + RESOURCE_BASE, RESOURCE_NOTE, RESOURCE_SKILL, SOURCE_LINK, SOURCE_UPLOAD, Document, + KnowledgeBase, Memory, Note, Share, Skill, SkillRevision, + chat_knowledge_bases, ) from lembas.db.models.setting import Setting from lembas.db.models.user import ( @@ -57,7 +59,7 @@ __all__ = [ "KIND_TEXT", "PRINCIPAL_GROUP", "PRINCIPAL_USER", - "RESOURCE_DOCUMENT", + "RESOURCE_BASE", "RESOURCE_NOTE", "RESOURCE_SKILL", "ROLE_ADMIN", @@ -73,6 +75,7 @@ __all__ = [ "Document", "Folder", "Group", + "KnowledgeBase", "Memory", "Message", "Model", @@ -83,6 +86,7 @@ __all__ = [ "Skill", "SkillRevision", "User", + "chat_knowledge_bases", "model_groups", "user_groups", ] diff --git a/src/lembas/db/models/chat.py b/src/lembas/db/models/chat.py index 720af82..8d2a8d5 100644 --- a/src/lembas/db/models/chat.py +++ b/src/lembas/db/models/chat.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any from sqlalchemy import Boolean, ForeignKey, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -10,6 +10,12 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship from lembas.db.base import Base, Timestamps, UUIDPrimaryKey from lembas.db.types import JSONDict, JSONList +if TYPE_CHECKING: + # Annotation only; SQLAlchemy resolves the name through its own registry at + # runtime, so there is no import cycle. A bare `Mapped[list]` would be read + # as a scalar and hand back None instead of []. + from lembas.db.models.library import KnowledgeBase + ROLE_SYSTEM = "system" ROLE_USER = "user" ROLE_ASSISTANT = "assistant" @@ -83,6 +89,11 @@ class Chat(UUIDPrimaryKey, Timestamps, Base): cascade="all, delete-orphan", order_by="Message.created_at", ) + # Which knowledge bases this chat draws on. None means "everything its owner + # can see"; naming some scopes the knowledge tool to those. + knowledge_bases: Mapped[list[KnowledgeBase]] = relationship( + "KnowledgeBase", secondary="chat_knowledge_bases" + ) def __repr__(self) -> str: return f"" diff --git a/src/lembas/db/models/library.py b/src/lembas/db/models/library.py index d97d35e..b347e41 100644 --- a/src/lembas/db/models/library.py +++ b/src/lembas/db/models/library.py @@ -23,7 +23,17 @@ hand round, and "share my memories with the team" is a question nobody asked. from __future__ import annotations -from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, Text, UniqueConstraint +from sqlalchemy import ( + Boolean, + Column, + ForeignKey, + Index, + Integer, + String, + Table, + Text, + UniqueConstraint, +) from sqlalchemy.orm import Mapped, mapped_column, relationship from lembas.db.base import Base, Timestamps, UUIDPrimaryKey @@ -39,13 +49,55 @@ SOURCE_LINK = "link" # Resource kinds that can be shared. Values are stored, so they are part of the # schema rather than an implementation detail. -RESOURCE_DOCUMENT = "document" +RESOURCE_BASE = "base" RESOURCE_NOTE = "note" RESOURCE_SKILL = "skill" PRINCIPAL_USER = "user" PRINCIPAL_GROUP = "group" +# Which knowledge bases a chat draws on. A chat with none searches everything +# its owner can see; a chat with some is scoped to those, which is the point -- +# "answer from the contract folder" is a different question from "answer from +# everything I have ever uploaded". +chat_knowledge_bases = Table( + "chat_knowledge_bases", + Base.metadata, + Column("chat_id", String(32), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True), + Column( + "base_id", + String(32), + ForeignKey("knowledge_bases.id", ondelete="CASCADE"), + primary_key=True, + ), +) + + +class KnowledgeBase(UUIDPrimaryKey, Timestamps, Base): + """A named collection of documents. + + Sharing lives here rather than on the individual document: "this folder is + the team's" is the granularity people actually think in, and per-document + grants would mean answering "who can see this?" by checking every file. + A document is visible to whoever can see the base it is in. + """ + + __tablename__ = "knowledge_bases" + __table_args__ = (UniqueConstraint("owner_id", "name"),) + + owner_id: Mapped[str] = mapped_column( + String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + name: Mapped[str] = mapped_column(String(200), nullable=False) + description: Mapped[str] = mapped_column(Text, default="") + + documents: Mapped[list[Document]] = relationship( + back_populates="base", cascade="all, delete-orphan" + ) + + def __repr__(self) -> str: + return f"" + class Document(UUIDPrimaryKey, Timestamps, Base): """One item in a knowledge library: a file, an image or a saved web page. @@ -61,6 +113,12 @@ class Document(UUIDPrimaryKey, Timestamps, Base): owner_id: Mapped[str] = mapped_column( String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True ) + # Nullable only so the column could be added to an existing table. The + # service always sets it, and a startup sweep files anything that predates + # bases into its owner's default -- see documents.sweep_unfiled. + base_id: Mapped[str | None] = mapped_column( + String(32), ForeignKey("knowledge_bases.id", ondelete="CASCADE"), index=True + ) title: Mapped[str] = mapped_column(String(300), nullable=False) description: Mapped[str] = mapped_column(Text, default="") @@ -82,6 +140,8 @@ class Document(UUIDPrimaryKey, Timestamps, Base): truncated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) extraction_error: Mapped[str] = mapped_column(Text, default="") + base: Mapped[KnowledgeBase] = relationship(back_populates="documents") + @property def is_image(self) -> bool: return self.kind == "image" diff --git a/src/lembas/main.py b/src/lembas/main.py index ea665b9..fa4992e 100644 --- a/src/lembas/main.py +++ b/src/lembas/main.py @@ -62,9 +62,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: try: from lembas.db.session import session_scope from lembas.services.files import sweep_orphans + from lembas.services.library.documents import sweep_unfiled with session_scope() as db: sweep_orphans(db) + # Documents that predate knowledge bases have nowhere to live until + # this runs; see services/library/documents.py. + sweep_unfiled(db) except Exception: # noqa: BLE001 - housekeeping must never block startup log.exception("orphaned upload sweep failed") diff --git a/src/lembas/services/chat.py b/src/lembas/services/chat.py index 8a5b80c..1edc03d 100644 --- a/src/lembas/services/chat.py +++ b/src/lembas/services/chat.py @@ -246,7 +246,7 @@ def build_request( # behaviour. See services/harness.py for why these are joined rather than # being two competing layers. system = harness_service.join( - harness_service.compose(db, user, tools), effective_system_prompt(db, chat) + harness_service.compose(db, user, tools, chat), effective_system_prompt(db, chat) ) body: dict[str, Any] = { diff --git a/src/lembas/services/generation.py b/src/lembas/services/generation.py index 7883d2a..037a69b 100644 --- a/src/lembas/services/generation.py +++ b/src/lembas/services/generation.py @@ -175,7 +175,7 @@ async def _run(generation: Generation) -> None: ) question = _question_from(payload) needs_title = not chat.title_generated - tool_context = tools_service.context_for(db, owner) + tool_context = tools_service.context_for(db, owner, chat) for round_number in range(tools_service.MAX_ROUNDS + 1): accumulator = tools_service.ToolCallAccumulator() diff --git a/src/lembas/services/harness.py b/src/lembas/services/harness.py index ef7059e..13670bb 100644 --- a/src/lembas/services/harness.py +++ b/src/lembas/services/harness.py @@ -103,6 +103,7 @@ def compose( db: DBSession, user: User | None, tools: list[dict[str, Any]] | None, + chat=None, ) -> str: """The operational preamble for this request, or "" when there is nothing to say.""" families = _families(tools or []) @@ -129,6 +130,16 @@ def compose( if block: parts += ["", "### What you know about this person", "", block] + # Naming the bases a chat is scoped to matters: without it the model cannot + # tell "there is nothing about this" from "I am only allowed to see the + # contracts folder", and phrases a miss as the former. + if "knowledge" in families and chat is not None and chat.knowledge_bases: + names = ", ".join(base.name for base in chat.knowledge_bases) + parts += [ + "", + f"Knowledge searches in this chat cover only: {names}.", + ] + if "skills" in families: index = skills_service.index_block(db, user) if index: diff --git a/src/lembas/services/library/documents.py b/src/lembas/services/library/documents.py index f4f0792..df2f947 100644 --- a/src/lembas/services/library/documents.py +++ b/src/lembas/services/library/documents.py @@ -18,7 +18,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session as DBSession from lembas.config import settings -from lembas.db.models import SOURCE_LINK, SOURCE_UPLOAD, Document, User +from lembas.db.models import SOURCE_LINK, SOURCE_UPLOAD, Document, KnowledgeBase, User from lembas.services import files as files_service from lembas.services import sharing from lembas.services.fetch import Fetched @@ -28,6 +28,10 @@ log = logging.getLogger(__name__) INDEX = "documents_fts" +# What a first base is called when one has to be invented -- on the first +# upload, or for documents that predate bases existing. +DEFAULT_BASE_NAME = "My documents" + # How much of a document's text a search result carries back to the model. A # whole 100-page extract would swallow the context window; this is enough to # judge relevance and to answer from, and `knowledge_get` fetches the rest. @@ -57,12 +61,109 @@ def stored_path(stored_name: str) -> Path | None: return path if path.is_file() else None +# --- Bases ------------------------------------------------------------------- +def visible_bases(db: DBSession, user: User | None): + return select(KnowledgeBase).where(sharing.visible_to(KnowledgeBase, user)) + + +def get_base(db: DBSession, base_id: str, user: User | None) -> KnowledgeBase | None: + base = db.get(KnowledgeBase, base_id) + if base is None or not sharing.can_read(db, base, user): + return None + return base + + +def create_base( + db: DBSession, *, owner: User, name: str, description: str = "" +) -> KnowledgeBase: + name = " ".join((name or "").split())[:200] or DEFAULT_BASE_NAME + existing = db.scalar( + select(KnowledgeBase).where( + KnowledgeBase.owner_id == owner.id, KnowledgeBase.name == name + ) + ) + if existing is not None: + raise ValueError(f"You already have a knowledge base called {name!r}.") + + base = KnowledgeBase(owner_id=owner.id, name=name, description=description.strip()[:2000]) + db.add(base) + db.commit() + return base + + +def default_base(db: DBSession, owner: User) -> KnowledgeBase: + """The base a document goes into when none was chosen. + + Made on demand rather than at registration, so an account that never uses + the library never grows an empty one. + """ + base = db.scalar( + select(KnowledgeBase) + .where(KnowledgeBase.owner_id == owner.id) + .order_by(KnowledgeBase.created_at) + ) + if base is not None: + return base + base = KnowledgeBase(owner_id=owner.id, name=DEFAULT_BASE_NAME) + db.add(base) + db.commit() + return base + + +def delete_base(db: DBSession, base: KnowledgeBase) -> None: + """Delete a base and everything in it. + + The documents go too -- a base is a place, not a label, and leaving its + contents behind with nowhere to live would need an "unfiled" concept that + exists only to hold the wreckage of deletes. + """ + for document in list(base.documents): + path = stored_path(document.stored_name) + if path is not None: + path.unlink(missing_ok=True) + sharing.forget_resource(db, base) + db.delete(base) + db.commit() + + +def sweep_unfiled(db: DBSession) -> int: + """File documents that predate knowledge bases into their owner's default. + + `Document.base_id` is nullable only because the column had to be added to a + table that already had rows. This is what makes "always set" true in + practice, and it runs at startup beside the orphaned-upload sweep. + """ + unfiled = list(db.scalars(select(Document).where(Document.base_id.is_(None)))) + if not unfiled: + return 0 + + bases: dict[str, KnowledgeBase] = {} + for document in unfiled: + owner = db.get(User, document.owner_id) + if owner is None: + continue + if owner.id not in bases: + bases[owner.id] = default_base(db, owner) + document.base_id = bases[owner.id].id + + db.commit() + log.info("filed %d document(s) that predated knowledge bases", len(unfiled)) + return len(unfiled) + + # --- Creating ---------------------------------------------------------------- def store_upload( - db: DBSession, *, owner: User, payload: bytes, filename: str, title: str = "" + db: DBSession, + *, + owner: User, + payload: bytes, + filename: str, + title: str = "", + base: KnowledgeBase | None = None, ) -> Document: """Add an uploaded file to the library. Raises files.FileError if unusable.""" prepared = files_service.prepare(payload, filename) + base = base or default_base(db, owner) stored_name = f"{secrets.token_hex(16)}{prepared.extension}" (library_dir() / stored_name).write_bytes(prepared.payload) @@ -70,6 +171,7 @@ def store_upload( display = files_service.safe_display_name(filename) document = Document( owner_id=owner.id, + base_id=base.id, title=(title.strip() or display)[:300], source=SOURCE_UPLOAD, filename=display, @@ -90,14 +192,18 @@ def store_upload( return document -def store_page(db: DBSession, *, owner: User, page: Fetched) -> Document: +def store_page( + db: DBSession, *, owner: User, page: Fetched, base: KnowledgeBase | None = None +) -> Document: """Add a fetched web page to the library. Saved as text rather than as the original HTML: the point of keeping it is what it said, and the markup would have to be reduced again on every read. """ + base = base or default_base(db, owner) document = Document( owner_id=owner.id, + base_id=base.id, title=page.title[:300] or page.url[:300], source=SOURCE_LINK, source_url=page.url, @@ -115,19 +221,41 @@ def store_page(db: DBSession, *, owner: User, page: Fetched) -> Document: # --- Reading ----------------------------------------------------------------- -def visible(db: DBSession, user: User | None): - return select(Document).where(sharing.visible_to(Document, user)) +def visible(db: DBSession, user: User | None, *, base_ids: list[str] | None = None): + """Documents this user may see, optionally narrowed to some bases. + + Visibility comes from the base, not the document: a document is readable by + whoever can read the base it lives in. That is the whole reason bases are + shareable and documents are not. + """ + condition = Document.base_id.in_( + select(KnowledgeBase.id).where(sharing.visible_to(KnowledgeBase, user)) + ) + query = select(Document).where(condition) + if base_ids: + # Still filtered by visibility above, so naming a base you cannot see + # returns nothing rather than granting access to it. + query = query.where(Document.base_id.in_(base_ids)) + return query def get(db: DBSession, document_id: str, user: User | None) -> Document | None: document = db.get(Document, document_id) - if document is None or not sharing.can_read(db, document, user): + if document is None: + return None + base = db.get(KnowledgeBase, document.base_id) if document.base_id else None + if base is None or not sharing.can_read(db, base, user): return None return document def search( - db: DBSession, user: User | None, needle: str, *, limit: int = 10 + db: DBSession, + user: User | None, + needle: str, + *, + limit: int = 10, + base_ids: list[str] | None = None, ) -> list[Document]: """Documents matching `needle` that this user may see, best match first. @@ -141,7 +269,9 @@ def search( order = {hit.id: position for position, hit in enumerate(hits)} rows = list( - db.scalars(visible(db, user).where(Document.id.in_(list(order)))) + db.scalars( + visible(db, user, base_ids=base_ids).where(Document.id.in_(list(order))) + ) ) rows.sort(key=lambda document: order.get(document.id, len(order))) return rows[:limit] @@ -160,7 +290,5 @@ def delete(db: DBSession, document: Document) -> None: path = stored_path(document.stored_name) if path is not None: path.unlink(missing_ok=True) - # Shares carry no foreign key to their resource, so nothing cascades. - sharing.forget_resource(db, document) db.delete(document) db.commit() diff --git a/src/lembas/services/sharing.py b/src/lembas/services/sharing.py index 9da65e4..e885063 100644 --- a/src/lembas/services/sharing.py +++ b/src/lembas/services/sharing.py @@ -3,6 +3,11 @@ One rule, in one place, for all three: you can see a resource if you own it, if it was shared with you by name, or if it was shared with a group you are in. +Documents are deliberately absent from that list. They are shared through the +knowledge base they belong to -- "this folder is the team's" is the granularity +people think in, and per-document grants would mean answering "who can see +this?" by checking every file. See services.library.documents.visible. + Everything that lists or searches a library store goes through `visible_to`. Writing the same condition into each query would work right up until one of them was written slightly differently, and the way that failure shows up is @@ -26,10 +31,10 @@ from sqlalchemy.orm import Session as DBSession from lembas.db.models import ( PRINCIPAL_GROUP, PRINCIPAL_USER, - RESOURCE_DOCUMENT, + RESOURCE_BASE, RESOURCE_NOTE, RESOURCE_SKILL, - Document, + KnowledgeBase, Note, Share, Skill, @@ -41,7 +46,7 @@ log = logging.getLogger(__name__) # The mapping between a model class and the string stored in Share. Kept here # so no caller has to remember which literal goes with which table. RESOURCE_TYPES: dict[Any, str] = { - Document: RESOURCE_DOCUMENT, + KnowledgeBase: RESOURCE_BASE, Note: RESOURCE_NOTE, Skill: RESOURCE_SKILL, } diff --git a/src/lembas/services/tools.py b/src/lembas/services/tools.py index 104ee38..f23a108 100644 --- a/src/lembas/services/tools.py +++ b/src/lembas/services/tools.py @@ -73,6 +73,9 @@ class ToolContext: owner_id: str search_config: dict[str, Any] = field(default_factory=dict) allow_private_fetch: bool = False + # Which knowledge bases this chat is scoped to. Empty means "everything the + # owner can see", which is what a chat with none attached should do. + base_ids: list[str] = field(default_factory=list) @dataclass @@ -171,7 +174,9 @@ async def _run_knowledge_search(context: ToolContext, args: dict[str, Any]) -> T with session_scope() as db: user = db.get(User, context.owner_id) - found = documents_service.search(db, user, query, limit=6) + found = documents_service.search( + db, user, query, limit=6, base_ids=context.base_ids + ) event = { "name": "knowledge_search", "query": query, @@ -677,11 +682,12 @@ def enabled_tools(db: DBSession, chat: Chat, user: User | None) -> list[dict[str return [tool.schema for tool in REGISTRY.values() if tool.family in families] -def context_for(db: DBSession, user: User | None) -> ToolContext: +def context_for(db: DBSession, user: User | None, chat: Chat | None = None) -> ToolContext: """The snapshot a running tool needs, taken while the session is open.""" return ToolContext( owner_id=user.id if user else "", search_config=settings_store.search(db), + base_ids=[base.id for base in chat.knowledge_bases] if chat is not None else [], ) diff --git a/src/lembas/web/static/css/app.css b/src/lembas/web/static/css/app.css index 06fd3d6..aeca9ed 100644 --- a/src/lembas/web/static/css/app.css +++ b/src/lembas/web/static/css/app.css @@ -231,6 +231,43 @@ button, input, textarea, select { box-shadow: 0 0 0 3px var(--accent-soft); } .input::placeholder, .textarea::placeholder { color: var(--ink-faint); } + +/* + File inputs. + + A file input is two things in one box -- a button the browser draws and the + chosen filename beside it -- and neither inherits anything useful. Left alone + with `.input`, the padding applies to the whole control so the button sits + hard against the left edge while the text floats off its centre line. + + So: no horizontal padding on the control, the button styled to the same + height as everything else and given the right border that separates it, and + the filename centred with line-height rather than flexbox, which file inputs + do not lay out reliably. +*/ +.input[type="file"] { + padding: 0 var(--control-px) 0 0; + line-height: calc(var(--control-h) - 2px); + cursor: pointer; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--ink-muted); +} +.input[type="file"]::file-selector-button { + height: calc(var(--control-h) - 2px); + margin: 0 var(--sp-3) 0 0; + padding: 0 var(--control-px); + border: 0; + border-right: 1px solid var(--border); + background: var(--surface-hover); + color: var(--ink); + font: inherit; + font-weight: 500; + cursor: pointer; + transition: background var(--transition-fast); +} +.input[type="file"]:hover::file-selector-button { background: var(--surface-active); } .input--mono, .textarea--mono { font-family: var(--font-mono); font-size: var(--text-xs); } .select { diff --git a/src/lembas/web/templates/chat/_knowledge_picker.html b/src/lembas/web/templates/chat/_knowledge_picker.html index a20e7e6..1c65fea 100644 --- a/src/lembas/web/templates/chat/_knowledge_picker.html +++ b/src/lembas/web/templates/chat/_knowledge_picker.html @@ -25,7 +25,7 @@ {{ document.title }} - {{ document.kind }} + {% if document.base %}{{ document.base.name }} · {% endif %}{{ document.kind }} {%- if document.pages %} · {{ document.pages }}p{% endif %} {%- if document.owner_id != user.id %} · shared{% endif %} diff --git a/src/lembas/web/templates/chat/index.html b/src/lembas/web/templates/chat/index.html index e87ec2d..c4dc057 100644 --- a/src/lembas/web/templates/chat/index.html +++ b/src/lembas/web/templates/chat/index.html @@ -68,6 +68,36 @@ {% endif %} + {% if knowledge_bases %} + {# Which bases this chat draws on. None ticked means everything you can + see, which is what a chat with nothing chosen should do. #} +
+ +
+ {# Always submitted, so unticking the last box still says something. + An absent checkbox carries no signal of its own. #} + +
+ {% for base in knowledge_bases %} + + {% endfor %} +
+
+

+ {% if attached_base_ids %} + Searches in this chat are limited to what is ticked. + {% else %} + Nothing ticked, so this chat can search everything in your + library. + {% endif %} +

+
+ {% endif %} + {% if can.get("chat.params") %}
diff --git a/src/lembas/web/templates/library/base_detail.html b/src/lembas/web/templates/library/base_detail.html new file mode 100644 index 0000000..c020087 --- /dev/null +++ b/src/lembas/web/templates/library/base_detail.html @@ -0,0 +1,126 @@ +{% extends "library/_layout.html" %} +{% from "_macros.html" import icon %} +{% set section = "knowledge" %} + +{% block title %}{{ base.name }} - LLeMbas{% endblock %} +{% block heading %}{{ base.name }}{% endblock %} + +{% block library_content %} +
+ + {{ icon("chevron-right", "icon--sm") }} All knowledge bases + +
+ +{% if is_owner %} +
+

Add to this base

+
+
+ +
+ + +

+ Images, PDFs and text. A PDF has its text read once, now. +

+
+ +
+ +
+ +
+ + +

+ Fetched now and kept as text, so it survives the page changing. +

+
+ +
+
+
+{% endif %} + +
+ + + {% if q %}Clear{% endif %} +
+ +{% if not documents %} +
+ {{ icon("archive", "empty__mark") }} +

{{ "Nothing found" if q else "Empty" }}

+

+ {% if q %}Nothing in this base matches “{{ q }}”. + {% else %}Add a file or a web page above.{% endif %} +

+
+{% else %} +
    + {% for document in documents %} +
  • +
    + + {{ document.title }} + +
    + {{ document.kind }} + {%- if document.pages %} · {{ document.pages }} page{{ '' if document.pages == 1 else 's' }}{% endif %} + {%- if document.size_bytes %} · {{ document.human_size }}{% endif %} + {%- if document.source_url %} · {{ document.source_url[:60] }}{% endif %} +
    + {% if document.description %} +
    {{ document.description }}
    + {% endif %} +
    +
    + {% if document.extraction_error %} + no text + {% endif %} +
    +
    • + {% endfor %} +
+{% include "library/_pager.html" %} +{% endif %} + +
+
+

This base

+
+
+ + +
+
+ + +
+
+
+ + {% include "library/_share.html" %} + + {% if is_owner %} +
+ + + +
+ {% endif %} +
+{% endblock %} diff --git a/src/lembas/web/templates/library/knowledge.html b/src/lembas/web/templates/library/knowledge.html index e5a76c2..0b48ddd 100644 --- a/src/lembas/web/templates/library/knowledge.html +++ b/src/lembas/web/templates/library/knowledge.html @@ -7,85 +7,60 @@ {% block library_content %}

- Documents, images and saved web pages you have collected. A model with the - knowledge tool searches these before it searches the web, and you can attach - any of them to a message. + Knowledge bases are collections of documents, images and saved web pages. Keep + them separate — one per subject, project or client — and a chat can be pointed + at just the ones it should draw on. Sharing happens here too: share a base and + everything in it comes with it.

-
-

Add

-
-
-
- - -

- Images, PDFs and text. PDFs have their text read once, now. -

-
- -
+{% if error %} +
{{ icon("warning", "alert__icon") }} {{ error }}
+{% endif %} -
-
- - -

- Fetched now and kept as text, so it survives the page changing. -

-
- -
-
-
- -
- - - {% if q %}Clear{% endif %} -
- -{% if not documents %} -
- {{ icon("archive", "empty__mark") }} -

{{ "Nothing found" if q else "The shelves are bare" }}

-

- {% if q %} - No document matches “{{ q }}”. - {% else %} - Add a file or a web page above and it becomes searchable — by you, and by - any model you have given the knowledge tool. - {% endif %} -

-
-{% else %} +{% if bases %}
    - {% for document in documents %} + {% for base in bases %}
  • - {{ document.title }} + {{ base.name }}
    - {{ document.kind }} - {%- if document.pages %} · {{ document.pages }} page{{ '' if document.pages == 1 else 's' }}{% endif %} - {%- if document.size_bytes %} · {{ document.human_size }}{% endif %} - {%- if document.source_url %} · {{ document.source_url[:60] }}{% endif %} + {{ counts.get(base.id, 0) }} document{{ '' if counts.get(base.id, 0) == 1 else 's' }} + {%- if base.description %} · {{ base.description }}{% endif %}
    - {% if document.description %} -
    {{ document.description }}
    - {% endif %}
    - {% if document.extraction_error %} - no text - {% endif %} - {% if document.owner_id != user.id %}shared{% endif %} + {% if base.owner_id != user.id %}shared with you{% endif %}
    • {% endfor %}
-{% include "library/_pager.html" %} +{% else %} +
+ {{ icon("archive", "empty__mark") }} +

No knowledge bases yet

+

+ Make one below, then put documents in it. A chat with no base attached + searches everything you have; a chat pointed at one searches only that. +

+
{% endif %} + +
+

New knowledge base

+
+
+
+ + +
+
+ + +
+
+ +
+
{% endblock %} diff --git a/src/lembas/web/templates/library/knowledge_detail.html b/src/lembas/web/templates/library/knowledge_detail.html index 285593f..ed480da 100644 --- a/src/lembas/web/templates/library/knowledge_detail.html +++ b/src/lembas/web/templates/library/knowledge_detail.html @@ -7,7 +7,9 @@ {% block library_content %}
- {{ icon("chevron-right", "icon--sm") }} All documents + + {{ icon("chevron-right", "icon--sm") }} Back to {{ document.base.name if document.base else "the base" }} +
@@ -28,6 +30,22 @@

+ {% if document.base %} +
+ + {# Moving a document changes who can see it, which is the point of bases. + Only bases this person can write to are offered. #} + +

Moving it changes who can see it.

+
+ {% endif %} +
Kind
{{ document.kind }}
{% if document.source_url %} @@ -50,8 +68,6 @@ {% endif %} - {% include "library/_share.html" %} - {% if is_owner %}
diff --git a/tests/test_sharing.py b/tests/test_sharing.py index 1ea6ffd..2142814 100644 --- a/tests/test_sharing.py +++ b/tests/test_sharing.py @@ -12,7 +12,6 @@ from sqlalchemy import select from lembas.db.models import ( PRINCIPAL_GROUP, PRINCIPAL_USER, - Document, Group, Note, Share, @@ -147,25 +146,25 @@ def test_forgetting_a_principal_drops_their_shares(db, people): def test_two_kinds_of_resource_do_not_collide(db, people): """One shares table across three resource types, so the type must be part - of the match -- otherwise a note and a document sharing an id would share - each other's access.""" + of the match -- otherwise a note and a base sharing an id would share each + other's access.""" note = _note(db, people["frodo"]) - document = documents_service.store_upload( - db, owner=people["frodo"], payload=b"hello", filename="a.txt" - ) + base = documents_service.create_base(db, owner=people["frodo"], name="Papers") sharing.set_grants(db, note, user_ids=[people["gollum"].id], group_ids=[]) assert sharing.can_read(db, note, people["gollum"]) - assert not sharing.can_read(db, document, people["gollum"]) + assert not sharing.can_read(db, base, people["gollum"]) -def test_resource_type_refuses_something_unshareable(db, people): - """Memory is deliberately not shareable: a record about a person is not - content to hand round.""" - from lembas.db.models import Memory +@pytest.mark.parametrize("unshareable", ["Memory", "Document"]) +def test_resource_type_refuses_something_unshareable(db, people, unshareable): + """Memory is not shareable at all -- a record about a person is not content + to hand round. A Document is shared through the base it lives in, so asking + to share one directly is a mistake worth catching loudly.""" + import lembas.db.models as models with pytest.raises(ValueError): - sharing.resource_type(Memory) + sharing.resource_type(getattr(models, unshareable)) # --- Through the search path ------------------------------------------------- @@ -180,20 +179,53 @@ def test_search_does_not_leak_across_owners(db, people): assert notes_service.search(db, people["gandalf"], "golden") == [] -def test_a_shared_document_is_findable_by_the_person_it_was_shared_with(db, people): +def test_sharing_a_base_shares_what_is_in_it(db, people): + """Documents are shared through their base. "This folder is the team's" is + the granularity people think in, and per-document grants would mean + answering "who can see this?" by checking every file.""" + base = documents_service.create_base(db, owner=people["frodo"], name="Trees") document = documents_service.store_upload( db, owner=people["frodo"], payload=b"The mallorn is a golden tree.", filename="tree.txt", + base=base, ) assert documents_service.search(db, people["gollum"], "mallorn") == [] - sharing.set_grants(db, document, user_ids=[people["gollum"].id], group_ids=[]) + sharing.set_grants(db, base, user_ids=[people["gollum"].id], group_ids=[]) found = documents_service.search(db, people["gollum"], "mallorn") assert [d.id for d in found] == [document.id] +def test_a_document_in_an_unshared_base_stays_private(db, people): + """Two bases, one shared: the other must not come with it.""" + shared = documents_service.create_base(db, owner=people["frodo"], name="Public") + private = documents_service.create_base(db, owner=people["frodo"], name="Private") + documents_service.store_upload( + db, owner=people["frodo"], payload=b"A mallorn tree.", filename="a.txt", base=shared + ) + documents_service.store_upload( + db, owner=people["frodo"], payload=b"A mallorn secret.", filename="b.txt", base=private + ) + sharing.set_grants(db, shared, user_ids=[people["gollum"].id], group_ids=[]) + + found = documents_service.search(db, people["gollum"], "mallorn") + assert [d.base_id for d in found] == [shared.id] + + +def test_scoping_to_a_base_cannot_be_used_to_reach_one(db, people): + """Naming a base you cannot see returns nothing rather than granting it.""" + private = documents_service.create_base(db, owner=people["frodo"], name="Private") + documents_service.store_upload( + db, owner=people["frodo"], payload=b"A mallorn tree.", filename="a.txt", base=private + ) + found = documents_service.search( + db, people["gollum"], "mallorn", base_ids=[private.id] + ) + assert found == [] + + def test_the_shares_table_records_what_was_asked_for(db, people): group = Group(name="Fellowship") db.add(group) @@ -222,10 +254,33 @@ def test_visibility_is_a_query_filter_not_a_python_loop(db, people): assert [n.title for n in rows] == ["Note 0", "Note 1"] -def test_documents_and_notes_use_the_same_rule(db, people): +def test_a_document_follows_its_base(db, people): document = documents_service.store_upload( db, owner=people["frodo"], payload=b"x", filename="a.txt" ) assert document in db.scalars(documents_service.visible(db, people["frodo"])) assert document not in db.scalars(documents_service.visible(db, people["gollum"])) - assert list(db.scalars(select(Document).where(sharing.visible_to(Document, None)))) == [] + assert list(db.scalars(documents_service.visible(db, None))) == [] + + +def test_an_uploaded_document_always_lands_in_a_base(db, people): + """base_id is nullable only so the column could be added to a table that + already had rows; the service never leaves it unset.""" + document = documents_service.store_upload( + db, owner=people["frodo"], payload=b"x", filename="a.txt" + ) + assert document.base_id is not None + + +def test_documents_predating_bases_are_filed_at_startup(db, people): + document = documents_service.store_upload( + db, owner=people["frodo"], payload=b"x", filename="a.txt" + ) + document.base_id = None + db.commit() + assert document not in db.scalars(documents_service.visible(db, people["frodo"])) + + assert documents_service.sweep_unfiled(db) == 1 + db.refresh(document) + assert document.base_id is not None + assert document in db.scalars(documents_service.visible(db, people["frodo"])) diff --git a/tests/test_tools.py b/tests/test_tools.py index 2dc1905..c53b2e4 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -166,6 +166,48 @@ def _context(**kwargs): return tools_service.ToolContext(owner_id="someone", **kwargs) +# --- Knowledge is scoped to the chat's bases --------------------------------- +async def test_knowledge_search_is_limited_to_the_attached_bases(db, user_id): + """"Answer from the contracts folder" is a different question from "answer + from everything I have ever uploaded".""" + from lembas.db.models import User + from lembas.services.library import documents as documents_service + + owner = db.get(User, user_id) + trees = documents_service.create_base(db, owner=owner, name="Trees") + contracts = documents_service.create_base(db, owner=owner, name="Contracts") + documents_service.store_upload( + db, owner=owner, payload=b"The mallorn is golden.", filename="a.txt", + title="Mallorn", base=trees, + ) + documents_service.store_upload( + db, owner=owner, payload=b"The mallorn clause is void.", filename="b.txt", + title="Clause", base=contracts, + ) + + everywhere = await tools_service.run_tool( + tools_service.ToolContext(owner_id=user_id), "knowledge_search", + '{"query": "mallorn"}', + ) + assert {r["title"] for r in everywhere.event["results"]} == {"Mallorn", "Clause"} + + scoped = await tools_service.run_tool( + tools_service.ToolContext(owner_id=user_id, base_ids=[contracts.id]), + "knowledge_search", + '{"query": "mallorn"}', + ) + assert [r["title"] for r in scoped.event["results"]] == ["Clause"] + + +def test_a_chat_with_no_bases_searches_everything(db, user_id): + """Empty means "everything the owner can see", not "nothing".""" + from lembas.db.models import User + + chat = _chat_with(db, user_id, capabilities={"tools": True}) + context = tools_service.context_for(db, db.get(User, user_id), chat) + assert context.base_ids == [] + + # --- Running one ------------------------------------------------------------- async def test_running_web_search_formats_results_for_the_model(monkeypatch): async def fake_run(_config, query, *, limit=None):