diff --git a/CLAUDE.md b/CLAUDE.md index 2bfbf2e..ea69ca5 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ lembas info # paths + counts, useful when confused lembas secret-key # generate LEMBAS_SECRET_KEY lembas create-admin # create or promote an admin -pytest # 428 tests, ~26s +pytest # 437 tests, ~27s # PLAN.md tracks what is and is not built ruff check . # lint (line length 100) python scripts/build_artwork.py # regenerate artwork (SVG + PWA icons; @@ -305,6 +305,23 @@ injected, body fetched by tool). The shape of each follows from how it reaches the model: a memory is capped short because it costs tokens on every request forever, a note is not injected because a dozen would fill the window. +**Documents live in knowledge bases, and the base is what is shared.** A +`Document` always belongs to a `KnowledgeBase`; visibility comes from the base, +never the document, which is why `Document` is absent from +`sharing.RESOURCE_TYPES` and `documents.visible()` filters on +`base_id IN (visible bases)`. Per-document grants would mean answering "who can +see this?" by checking every file. `Document.base_id` is nullable only because +the column had to be added to a table that already had rows; +`documents.sweep_unfiled()` runs at startup and files anything predating bases +into its owner's default. + +**A chat attached to bases is scoped to them.** `Chat.knowledge_bases` is +many-to-many; empty means "everything the owner can see", not "nothing". +`tools.context_for(db, user, chat)` carries the ids and `knowledge_search` +filters on them — and the harness names the bases, because otherwise the model +cannot tell "there is nothing about this" from "I am only allowed to see the +contracts folder". + **Sharing goes through one helper, and admins do not bypass it.** `services/sharing.py:visible_to()` is the only definition of who can see a library item, and every listing and tool uses it. `permissions.resolve` gives an diff --git a/PLAN.md b/PLAN.md index 3e71e56..8ccefc1 100644 --- a/PLAN.md +++ b/PLAN.md @@ -6,7 +6,7 @@ that would be expensive to revisit. Kept current as work lands; the detail of **Status:** usable daily. Streaming chat, attachments, reasoning, tool calling with web search, a knowledge library, notes, memory and skills, speech in and -out, users and groups, model administration, installable as an app. 428 tests, +out, users and groups, model administration, installable as an app. 437 tests, `ruff` clean. --- @@ -59,8 +59,11 @@ be a different project, not a refactor. the next turn, for the same reasons reasoning is not ### The library -- [x] **Knowledge** — documents, images and saved web pages, ingested through - the same pipeline as chat attachments, searched with SQLite FTS5 +- [x] **Knowledge bases** — documents, images and saved web pages, grouped into + named collections and ingested through the same pipeline as chat + attachments, searched with SQLite FTS5 +- [x] A chat can be pointed at particular bases, so "answer from the contracts + folder" is a different question from "answer from everything I have" - [x] **Notes** — longer things the model writes down and searches later; editable by hand, because they are yours - [x] **Memory** — short facts, injected on every turn to a budget rather than @@ -70,8 +73,9 @@ be a different project, not a refactor. - [x] A model may write and revise its own notes, memories and skills. Every skill revision is kept, attributed and revertible — the safety story is a record and a way back, not a gate -- [x] **Sharing** — any of the three can be shared with a group or with named - people, read-only. One visibility rule, and administrators do not bypass it +- [x] **Sharing** — a knowledge base, a note or a skill can be shared with a + group or with named people, read-only. One visibility rule, and + administrators do not bypass it. Documents are shared through their base - [x] **The harness** — an operational prompt assembled from what a model actually has, so the tools get used rather than ignored - [x] Attach menu: file, image, a web page fetched on the spot, or a document diff --git a/README.md b/README.md index af465fd..7e274d7 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,8 @@ runtime. Clone it, `pip install -e .`, run it. any OpenAI-compatible audio endpoint (whisper.cpp, Speaches, Kokoro…). Each person picks their own voice - **A library** — four places a model can reach for. **Knowledge**: documents, - images and web pages you collect, searched before the web. **Notes**: longer + images and web pages you collect, grouped into named bases so a chat can be + pointed at just the right one, searched before the web. **Notes**: longer things it writes down and finds again later. **Memory**: short facts about you, in front of it on every turn. **Skills**: saved procedures it can follow, and write. All of it visible and editable by you, and shareable with a group or a @@ -143,6 +144,11 @@ model: give it the tools it should have under **Admin → Models**, where `tools` decides whether a tool list may be sent at all and the built-in tools are chosen one by one. +Knowledge is organised into **bases** — one per subject, project or client. A +chat with no base attached searches everything you have; tick some in the chat's +settings panel and it searches only those. Sharing happens at the base: share it +and everything in it comes too, read-only. + Search is SQLite's FTS5 — keyword matching with BM25 ranking, no embedding service to run and nothing that stops working offline. It will not match a paraphrase, so a line of description on a document is worth writing. diff --git a/src/lembas/api/chats.py b/src/lembas/api/chats.py index d3d7dde..59c127b 100644 --- a/src/lembas/api/chats.py +++ b/src/lembas/api/chats.py @@ -484,6 +484,26 @@ async def update_chat(request: Request, db: Db, user: RequiredUser, chat_id: str ) chat.system_prompt = str(form["system_prompt"]).strip()[:8000] + if "knowledge_base_ids" in form: + # Sent as a single field even when empty, so that clearing every box + # actually clears the attachment -- absent checkboxes carry no signal of + # their own, which is the same trap update_chat exists to avoid. + from lembas.db.models import KnowledgeBase + from lembas.services.library import documents as documents_service + + wanted = [value for value in form.getlist("knowledge_base_ids") if value] + chat.knowledge_bases = ( + list( + db.scalars( + documents_service.visible_bases(db, user).where( + KnowledgeBase.id.in_(wanted) + ) + ) + ) + if wanted + else [] + ) + submitted_params = {name: form[name] for name in _PARAM_RANGES if name in form} if submitted_params: if not allowed.get("chat.params"): diff --git a/src/lembas/api/library.py b/src/lembas/api/library.py index e8f37fd..4a85d7f 100644 --- a/src/lembas/api/library.py +++ b/src/lembas/api/library.py @@ -27,6 +27,7 @@ from lembas.db.models import ( PRINCIPAL_USER, Document, Group, + KnowledgeBase, Note, Skill, SkillRevision, @@ -92,33 +93,54 @@ async def library_home(user: RequiredUser): # --- Knowledge --------------------------------------------------------------- +# Route order matters: /library/knowledge/document/{id} must be registered +# before /library/knowledge/{base_id}, or "document" is parsed as a base id. +# FastAPI matches in registration order and this has bitten before. @router.get("/library/knowledge") -async def knowledge_list( - request: Request, db: Db, user: RequiredUser, q: str = "", page: int = 1, saved: str = "" -): - if q.strip(): - # Search returns best-match order and its own limit, so it is not paged. - rows = documents_service.search(db, user, q, limit=PAGE_SIZE) - pager = {"page": 1, "pages": 1, "total": len(rows)} - else: - rows, pager = _page( - db, documents_service.visible(db, user).order_by(Document.created_at.desc()), page +async def knowledge_list(request: Request, db: Db, user: RequiredUser, error: str = ""): + """The bases, not the documents. A library is a set of places first.""" + bases = list(db.scalars(documents_service.visible_bases(db, user).order_by(KnowledgeBase.name))) + counts = { + base.id: db.scalar( + select(func.count()).select_from(Document).where(Document.base_id == base.id) ) + or 0 + for base in bases + } return render( request, "library/knowledge.html", { "section": "knowledge", - "documents": rows, - "q": q, - "pager": pager, - "saved": saved, + "bases": bases, + "counts": counts, + "error": error, **sidebar_context(db, user), }, ) -@router.get("/library/knowledge/{document_id}") +@router.post("/api/library/bases") +async def create_base( + db: Db, user: RequiredUser, name: str = Form(""), description: str = Form("") +) -> Response: + try: + base = documents_service.create_base( + db, owner=user, name=name, description=description + ) + except ValueError as exc: + from urllib.parse import quote + + return RedirectResponse( + f"/library/knowledge?error={quote(str(exc))}", + status_code=status.HTTP_303_SEE_OTHER, + ) + return RedirectResponse( + f"/library/knowledge/{base.id}", status_code=status.HTTP_303_SEE_OTHER + ) + + +@router.get("/library/knowledge/document/{document_id}") async def knowledge_detail(request: Request, db: Db, user: RequiredUser, document_id: str): document = documents_service.get(db, document_id, user) if document is None: @@ -129,38 +151,129 @@ async def knowledge_detail(request: Request, db: Db, user: RequiredUser, documen { "section": "knowledge", "document": document, - **_shared_context(db, user, document), + "is_owner": sharing.can_write(document, user), + # Only bases this person owns: moving a document into one they can + # merely read would hand it to that base's owner. + "user_bases": list( + db.scalars( + select(KnowledgeBase) + .where(KnowledgeBase.owner_id == user.id) + .order_by(KnowledgeBase.name) + ) + ), **sidebar_context(db, user), }, ) +@router.get("/library/knowledge/{base_id}") +async def base_detail( + request: Request, db: Db, user: RequiredUser, base_id: str, q: str = "", page: int = 1 +): + base = documents_service.get_base(db, base_id, user) + if base is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That knowledge base is not available.") + + if q.strip(): + rows = documents_service.search(db, user, q, limit=PAGE_SIZE, base_ids=[base.id]) + pager = {"page": 1, "pages": 1, "total": len(rows)} + else: + rows, pager = _page( + db, + documents_service.visible(db, user, base_ids=[base.id]).order_by( + Document.created_at.desc() + ), + page, + ) + return render( + request, + "library/base_detail.html", + { + "section": "knowledge", + "base": base, + "documents": rows, + "q": q, + "pager": pager, + **_shared_context(db, user, base), + **sidebar_context(db, user), + }, + ) + + +@router.post("/api/library/bases/{base_id}") +async def update_base(request: Request, db: Db, user: RequiredUser, base_id: str) -> Response: + base = documents_service.get_base(db, base_id, user) + if base is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That knowledge base is not available.") + if not sharing.can_write(base, user): + raise HTTPException(status.HTTP_403_FORBIDDEN, "That base is not yours to change.") + + form = await request.form() + name = " ".join(str(form.get("name", "")).split())[:200] + if name: + base.name = name + base.description = str(form.get("description", "")).strip()[:2000] + db.commit() + _apply_shares(db, user, base, form) + return RedirectResponse( + f"/library/knowledge/{base.id}", status_code=status.HTTP_303_SEE_OTHER + ) + + +@router.post("/api/library/bases/{base_id}/delete") +async def delete_base(db: Db, user: RequiredUser, base_id: str) -> Response: + base = documents_service.get_base(db, base_id, user) + if base is None or not sharing.can_write(base, user): + raise HTTPException(status.HTTP_404_NOT_FOUND, "That knowledge base is not available.") + documents_service.delete_base(db, base) + return RedirectResponse("/library/knowledge", status_code=status.HTTP_303_SEE_OTHER) + + @router.post("/api/library/documents") async def upload_document( - db: Db, user: RequiredUser, file: UploadFile = File(...), title: str = Form("") + db: Db, + user: RequiredUser, + file: UploadFile = File(...), + title: str = Form(""), + base_id: str = Form(""), ) -> Response: + base = documents_service.get_base(db, base_id, user) if base_id else None + if base is not None and not sharing.can_write(base, user): + raise HTTPException(status.HTTP_403_FORBIDDEN, "That base is not yours to add to.") + payload = await file.read(files_service.MAX_UPLOAD_BYTES + 1) try: document = documents_service.store_upload( - db, owner=user, payload=payload, filename=file.filename or "file", title=title + db, + owner=user, + payload=payload, + filename=file.filename or "file", + title=title, + base=base, ) except files_service.FileError as exc: raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc return RedirectResponse( - f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER + f"/library/knowledge/{document.base_id}", status_code=status.HTTP_303_SEE_OTHER ) @router.post("/api/library/documents/link") -async def save_link(db: Db, user: RequiredUser, url: str = Form(...)) -> Response: +async def save_link( + db: Db, user: RequiredUser, url: str = Form(...), base_id: str = Form("") +) -> Response: + base = documents_service.get_base(db, base_id, user) if base_id else None + if base is not None and not sharing.can_write(base, user): + raise HTTPException(status.HTTP_403_FORBIDDEN, "That base is not yours to add to.") + config = settings_store.search(db) try: page = await fetch(url, allow_private=bool(config.get("allow_private_fetch"))) except FetchError as exc: raise HTTPException(status.HTTP_400_BAD_REQUEST, exc.message) from exc - document = documents_service.store_page(db, owner=user, page=page) + document = documents_service.store_page(db, owner=user, page=page, base=base) return RedirectResponse( - f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER + f"/library/knowledge/{document.base_id}", status_code=status.HTTP_303_SEE_OTHER ) @@ -177,10 +290,18 @@ async def update_document( form = await request.form() document.title = str(form.get("title", document.title)).strip()[:300] or document.title document.description = str(form.get("description", "")).strip()[:2000] + + # Moving between bases changes who can see it, which is the whole point of + # bases -- so the destination has to be one this person can write to. + wanted = str(form.get("base_id", "")).strip() + if wanted and wanted != document.base_id: + destination = documents_service.get_base(db, wanted, user) + if destination is not None and sharing.can_write(destination, user): + document.base_id = destination.id + db.commit() - _apply_shares(db, user, document, form) return RedirectResponse( - f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER + f"/library/knowledge/document/{document.id}", status_code=status.HTTP_303_SEE_OTHER ) @@ -189,8 +310,11 @@ async def delete_document(db: Db, user: RequiredUser, document_id: str) -> Respo document = documents_service.get(db, document_id, user) if document is None or not sharing.can_write(document, user): raise HTTPException(status.HTTP_404_NOT_FOUND, "That document is not available.") + base_id = document.base_id documents_service.delete(db, document) - return RedirectResponse("/library/knowledge", status_code=status.HTTP_303_SEE_OTHER) + return RedirectResponse( + f"/library/knowledge/{base_id}", status_code=status.HTTP_303_SEE_OTHER + ) @router.get("/api/library/documents/{document_id}/content") diff --git a/src/lembas/api/pages.py b/src/lembas/api/pages.py index 7ec32a0..79a79d2 100644 --- a/src/lembas/api/pages.py +++ b/src/lembas/api/pages.py @@ -8,11 +8,12 @@ from sqlalchemy import select from sqlalchemy.orm import Session as DBSession from lembas.api.deps import Db, RequiredUser -from lembas.db.models import Chat, Folder, Message, User +from lembas.db.models import Chat, Folder, KnowledgeBase, Message, User from lembas.security import permissions from lembas.services import audio as audio_service from lembas.services import chat as chat_service from lembas.services import settings_store +from lembas.services.library import documents as documents_service from lembas.services.markdown import render_markdown from lembas.web.templating import STATIC_DIR, render @@ -42,6 +43,19 @@ def _chat_context(db: DBSession, user: User, chat: Chat | None) -> dict: # may not be the model the chat is set to now. Keyed by model_id, the # denormalised value stored on each message. "models_by_id": {m.model_id: m for m in models}, + # Offered in the chat settings panel so a conversation can be pointed at + # particular bases. Empty when the reader has none, and the panel then + # shows nothing rather than an empty control. + "knowledge_bases": ( + list( + db.scalars( + documents_service.visible_bases(db, user).order_by(KnowledgeBase.name) + ) + ) + if permissions.has(db, user, "library.use") + else [] + ), + "attached_base_ids": [base.id for base in chat.knowledge_bases] if chat else [], **audio_service.template_flags(db, user), } @@ -223,8 +237,6 @@ async def chat_detail(request: Request, db: Db, user: RequiredUser, chat_id: str if current is not None and (current.system_prompt or "").strip(): inherited, inherited_from = current.system_prompt.strip(), "model" else: - from lembas.services import settings_store - instance_prompt = (settings_store.get(db, "system_prompt") or "").strip() if instance_prompt: inherited, inherited_from = instance_prompt, "instance" diff --git a/src/lembas/db/models/__init__.py b/src/lembas/db/models/__init__.py index c6edaeb..8634c11 100644 --- a/src/lembas/db/models/__init__.py +++ b/src/lembas/db/models/__init__.py @@ -26,17 +26,19 @@ from lembas.db.models.library import ( AUTHOR_USER, PRINCIPAL_GROUP, PRINCIPAL_USER, - RESOURCE_DOCUMENT, + RESOURCE_BASE, RESOURCE_NOTE, RESOURCE_SKILL, SOURCE_LINK, SOURCE_UPLOAD, Document, + KnowledgeBase, Memory, Note, Share, Skill, SkillRevision, + chat_knowledge_bases, ) from lembas.db.models.setting import Setting from lembas.db.models.user import ( @@ -57,7 +59,7 @@ __all__ = [ "KIND_TEXT", "PRINCIPAL_GROUP", "PRINCIPAL_USER", - "RESOURCE_DOCUMENT", + "RESOURCE_BASE", "RESOURCE_NOTE", "RESOURCE_SKILL", "ROLE_ADMIN", @@ -73,6 +75,7 @@ __all__ = [ "Document", "Folder", "Group", + "KnowledgeBase", "Memory", "Message", "Model", @@ -83,6 +86,7 @@ __all__ = [ "Skill", "SkillRevision", "User", + "chat_knowledge_bases", "model_groups", "user_groups", ] diff --git a/src/lembas/db/models/chat.py b/src/lembas/db/models/chat.py index 720af82..8d2a8d5 100644 --- a/src/lembas/db/models/chat.py +++ b/src/lembas/db/models/chat.py @@ -2,7 +2,7 @@ from __future__ import annotations -from typing import Any +from typing import TYPE_CHECKING, Any from sqlalchemy import Boolean, ForeignKey, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column, relationship @@ -10,6 +10,12 @@ from sqlalchemy.orm import Mapped, mapped_column, relationship from lembas.db.base import Base, Timestamps, UUIDPrimaryKey from lembas.db.types import JSONDict, JSONList +if TYPE_CHECKING: + # Annotation only; SQLAlchemy resolves the name through its own registry at + # runtime, so there is no import cycle. A bare `Mapped[list]` would be read + # as a scalar and hand back None instead of []. + from lembas.db.models.library import KnowledgeBase + ROLE_SYSTEM = "system" ROLE_USER = "user" ROLE_ASSISTANT = "assistant" @@ -83,6 +89,11 @@ class Chat(UUIDPrimaryKey, Timestamps, Base): cascade="all, delete-orphan", order_by="Message.created_at", ) + # Which knowledge bases this chat draws on. None means "everything its owner + # can see"; naming some scopes the knowledge tool to those. + knowledge_bases: Mapped[list[KnowledgeBase]] = relationship( + "KnowledgeBase", secondary="chat_knowledge_bases" + ) def __repr__(self) -> str: return f"" diff --git a/src/lembas/db/models/library.py b/src/lembas/db/models/library.py index d97d35e..b347e41 100644 --- a/src/lembas/db/models/library.py +++ b/src/lembas/db/models/library.py @@ -23,7 +23,17 @@ hand round, and "share my memories with the team" is a question nobody asked. from __future__ import annotations -from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, Text, UniqueConstraint +from sqlalchemy import ( + Boolean, + Column, + ForeignKey, + Index, + Integer, + String, + Table, + Text, + UniqueConstraint, +) from sqlalchemy.orm import Mapped, mapped_column, relationship from lembas.db.base import Base, Timestamps, UUIDPrimaryKey @@ -39,13 +49,55 @@ SOURCE_LINK = "link" # Resource kinds that can be shared. Values are stored, so they are part of the # schema rather than an implementation detail. -RESOURCE_DOCUMENT = "document" +RESOURCE_BASE = "base" RESOURCE_NOTE = "note" RESOURCE_SKILL = "skill" PRINCIPAL_USER = "user" PRINCIPAL_GROUP = "group" +# Which knowledge bases a chat draws on. A chat with none searches everything +# its owner can see; a chat with some is scoped to those, which is the point -- +# "answer from the contract folder" is a different question from "answer from +# everything I have ever uploaded". +chat_knowledge_bases = Table( + "chat_knowledge_bases", + Base.metadata, + Column("chat_id", String(32), ForeignKey("chats.id", ondelete="CASCADE"), primary_key=True), + Column( + "base_id", + String(32), + ForeignKey("knowledge_bases.id", ondelete="CASCADE"), + primary_key=True, + ), +) + + +class KnowledgeBase(UUIDPrimaryKey, Timestamps, Base): + """A named collection of documents. + + Sharing lives here rather than on the individual document: "this folder is + the team's" is the granularity people actually think in, and per-document + grants would mean answering "who can see this?" by checking every file. + A document is visible to whoever can see the base it is in. + """ + + __tablename__ = "knowledge_bases" + __table_args__ = (UniqueConstraint("owner_id", "name"),) + + owner_id: Mapped[str] = mapped_column( + String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + name: Mapped[str] = mapped_column(String(200), nullable=False) + description: Mapped[str] = mapped_column(Text, default="") + + documents: Mapped[list[Document]] = relationship( + back_populates="base", cascade="all, delete-orphan" + ) + + def __repr__(self) -> str: + return f"" + class Document(UUIDPrimaryKey, Timestamps, Base): """One item in a knowledge library: a file, an image or a saved web page. @@ -61,6 +113,12 @@ class Document(UUIDPrimaryKey, Timestamps, Base): owner_id: Mapped[str] = mapped_column( String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True ) + # Nullable only so the column could be added to an existing table. The + # service always sets it, and a startup sweep files anything that predates + # bases into its owner's default -- see documents.sweep_unfiled. + base_id: Mapped[str | None] = mapped_column( + String(32), ForeignKey("knowledge_bases.id", ondelete="CASCADE"), index=True + ) title: Mapped[str] = mapped_column(String(300), nullable=False) description: Mapped[str] = mapped_column(Text, default="") @@ -82,6 +140,8 @@ class Document(UUIDPrimaryKey, Timestamps, Base): truncated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) extraction_error: Mapped[str] = mapped_column(Text, default="") + base: Mapped[KnowledgeBase] = relationship(back_populates="documents") + @property def is_image(self) -> bool: return self.kind == "image" diff --git a/src/lembas/main.py b/src/lembas/main.py index ea665b9..fa4992e 100644 --- a/src/lembas/main.py +++ b/src/lembas/main.py @@ -62,9 +62,13 @@ async def lifespan(app: FastAPI) -> AsyncIterator[None]: try: from lembas.db.session import session_scope from lembas.services.files import sweep_orphans + from lembas.services.library.documents import sweep_unfiled with session_scope() as db: sweep_orphans(db) + # Documents that predate knowledge bases have nowhere to live until + # this runs; see services/library/documents.py. + sweep_unfiled(db) except Exception: # noqa: BLE001 - housekeeping must never block startup log.exception("orphaned upload sweep failed") diff --git a/src/lembas/services/chat.py b/src/lembas/services/chat.py index 8a5b80c..1edc03d 100644 --- a/src/lembas/services/chat.py +++ b/src/lembas/services/chat.py @@ -246,7 +246,7 @@ def build_request( # behaviour. See services/harness.py for why these are joined rather than # being two competing layers. system = harness_service.join( - harness_service.compose(db, user, tools), effective_system_prompt(db, chat) + harness_service.compose(db, user, tools, chat), effective_system_prompt(db, chat) ) body: dict[str, Any] = { diff --git a/src/lembas/services/generation.py b/src/lembas/services/generation.py index 7883d2a..037a69b 100644 --- a/src/lembas/services/generation.py +++ b/src/lembas/services/generation.py @@ -175,7 +175,7 @@ async def _run(generation: Generation) -> None: ) question = _question_from(payload) needs_title = not chat.title_generated - tool_context = tools_service.context_for(db, owner) + tool_context = tools_service.context_for(db, owner, chat) for round_number in range(tools_service.MAX_ROUNDS + 1): accumulator = tools_service.ToolCallAccumulator() diff --git a/src/lembas/services/harness.py b/src/lembas/services/harness.py index ef7059e..13670bb 100644 --- a/src/lembas/services/harness.py +++ b/src/lembas/services/harness.py @@ -103,6 +103,7 @@ def compose( db: DBSession, user: User | None, tools: list[dict[str, Any]] | None, + chat=None, ) -> str: """The operational preamble for this request, or "" when there is nothing to say.""" families = _families(tools or []) @@ -129,6 +130,16 @@ def compose( if block: parts += ["", "### What you know about this person", "", block] + # Naming the bases a chat is scoped to matters: without it the model cannot + # tell "there is nothing about this" from "I am only allowed to see the + # contracts folder", and phrases a miss as the former. + if "knowledge" in families and chat is not None and chat.knowledge_bases: + names = ", ".join(base.name for base in chat.knowledge_bases) + parts += [ + "", + f"Knowledge searches in this chat cover only: {names}.", + ] + if "skills" in families: index = skills_service.index_block(db, user) if index: diff --git a/src/lembas/services/library/documents.py b/src/lembas/services/library/documents.py index f4f0792..df2f947 100644 --- a/src/lembas/services/library/documents.py +++ b/src/lembas/services/library/documents.py @@ -18,7 +18,7 @@ from sqlalchemy import select from sqlalchemy.orm import Session as DBSession from lembas.config import settings -from lembas.db.models import SOURCE_LINK, SOURCE_UPLOAD, Document, User +from lembas.db.models import SOURCE_LINK, SOURCE_UPLOAD, Document, KnowledgeBase, User from lembas.services import files as files_service from lembas.services import sharing from lembas.services.fetch import Fetched @@ -28,6 +28,10 @@ log = logging.getLogger(__name__) INDEX = "documents_fts" +# What a first base is called when one has to be invented -- on the first +# upload, or for documents that predate bases existing. +DEFAULT_BASE_NAME = "My documents" + # How much of a document's text a search result carries back to the model. A # whole 100-page extract would swallow the context window; this is enough to # judge relevance and to answer from, and `knowledge_get` fetches the rest. @@ -57,12 +61,109 @@ def stored_path(stored_name: str) -> Path | None: return path if path.is_file() else None +# --- Bases ------------------------------------------------------------------- +def visible_bases(db: DBSession, user: User | None): + return select(KnowledgeBase).where(sharing.visible_to(KnowledgeBase, user)) + + +def get_base(db: DBSession, base_id: str, user: User | None) -> KnowledgeBase | None: + base = db.get(KnowledgeBase, base_id) + if base is None or not sharing.can_read(db, base, user): + return None + return base + + +def create_base( + db: DBSession, *, owner: User, name: str, description: str = "" +) -> KnowledgeBase: + name = " ".join((name or "").split())[:200] or DEFAULT_BASE_NAME + existing = db.scalar( + select(KnowledgeBase).where( + KnowledgeBase.owner_id == owner.id, KnowledgeBase.name == name + ) + ) + if existing is not None: + raise ValueError(f"You already have a knowledge base called {name!r}.") + + base = KnowledgeBase(owner_id=owner.id, name=name, description=description.strip()[:2000]) + db.add(base) + db.commit() + return base + + +def default_base(db: DBSession, owner: User) -> KnowledgeBase: + """The base a document goes into when none was chosen. + + Made on demand rather than at registration, so an account that never uses + the library never grows an empty one. + """ + base = db.scalar( + select(KnowledgeBase) + .where(KnowledgeBase.owner_id == owner.id) + .order_by(KnowledgeBase.created_at) + ) + if base is not None: + return base + base = KnowledgeBase(owner_id=owner.id, name=DEFAULT_BASE_NAME) + db.add(base) + db.commit() + return base + + +def delete_base(db: DBSession, base: KnowledgeBase) -> None: + """Delete a base and everything in it. + + The documents go too -- a base is a place, not a label, and leaving its + contents behind with nowhere to live would need an "unfiled" concept that + exists only to hold the wreckage of deletes. + """ + for document in list(base.documents): + path = stored_path(document.stored_name) + if path is not None: + path.unlink(missing_ok=True) + sharing.forget_resource(db, base) + db.delete(base) + db.commit() + + +def sweep_unfiled(db: DBSession) -> int: + """File documents that predate knowledge bases into their owner's default. + + `Document.base_id` is nullable only because the column had to be added to a + table that already had rows. This is what makes "always set" true in + practice, and it runs at startup beside the orphaned-upload sweep. + """ + unfiled = list(db.scalars(select(Document).where(Document.base_id.is_(None)))) + if not unfiled: + return 0 + + bases: dict[str, KnowledgeBase] = {} + for document in unfiled: + owner = db.get(User, document.owner_id) + if owner is None: + continue + if owner.id not in bases: + bases[owner.id] = default_base(db, owner) + document.base_id = bases[owner.id].id + + db.commit() + log.info("filed %d document(s) that predated knowledge bases", len(unfiled)) + return len(unfiled) + + # --- Creating ---------------------------------------------------------------- def store_upload( - db: DBSession, *, owner: User, payload: bytes, filename: str, title: str = "" + db: DBSession, + *, + owner: User, + payload: bytes, + filename: str, + title: str = "", + base: KnowledgeBase | None = None, ) -> Document: """Add an uploaded file to the library. Raises files.FileError if unusable.""" prepared = files_service.prepare(payload, filename) + base = base or default_base(db, owner) stored_name = f"{secrets.token_hex(16)}{prepared.extension}" (library_dir() / stored_name).write_bytes(prepared.payload) @@ -70,6 +171,7 @@ def store_upload( display = files_service.safe_display_name(filename) document = Document( owner_id=owner.id, + base_id=base.id, title=(title.strip() or display)[:300], source=SOURCE_UPLOAD, filename=display, @@ -90,14 +192,18 @@ def store_upload( return document -def store_page(db: DBSession, *, owner: User, page: Fetched) -> Document: +def store_page( + db: DBSession, *, owner: User, page: Fetched, base: KnowledgeBase | None = None +) -> Document: """Add a fetched web page to the library. Saved as text rather than as the original HTML: the point of keeping it is what it said, and the markup would have to be reduced again on every read. """ + base = base or default_base(db, owner) document = Document( owner_id=owner.id, + base_id=base.id, title=page.title[:300] or page.url[:300], source=SOURCE_LINK, source_url=page.url, @@ -115,19 +221,41 @@ def store_page(db: DBSession, *, owner: User, page: Fetched) -> Document: # --- Reading ----------------------------------------------------------------- -def visible(db: DBSession, user: User | None): - return select(Document).where(sharing.visible_to(Document, user)) +def visible(db: DBSession, user: User | None, *, base_ids: list[str] | None = None): + """Documents this user may see, optionally narrowed to some bases. + + Visibility comes from the base, not the document: a document is readable by + whoever can read the base it lives in. That is the whole reason bases are + shareable and documents are not. + """ + condition = Document.base_id.in_( + select(KnowledgeBase.id).where(sharing.visible_to(KnowledgeBase, user)) + ) + query = select(Document).where(condition) + if base_ids: + # Still filtered by visibility above, so naming a base you cannot see + # returns nothing rather than granting access to it. + query = query.where(Document.base_id.in_(base_ids)) + return query def get(db: DBSession, document_id: str, user: User | None) -> Document | None: document = db.get(Document, document_id) - if document is None or not sharing.can_read(db, document, user): + if document is None: + return None + base = db.get(KnowledgeBase, document.base_id) if document.base_id else None + if base is None or not sharing.can_read(db, base, user): return None return document def search( - db: DBSession, user: User | None, needle: str, *, limit: int = 10 + db: DBSession, + user: User | None, + needle: str, + *, + limit: int = 10, + base_ids: list[str] | None = None, ) -> list[Document]: """Documents matching `needle` that this user may see, best match first. @@ -141,7 +269,9 @@ def search( order = {hit.id: position for position, hit in enumerate(hits)} rows = list( - db.scalars(visible(db, user).where(Document.id.in_(list(order)))) + db.scalars( + visible(db, user, base_ids=base_ids).where(Document.id.in_(list(order))) + ) ) rows.sort(key=lambda document: order.get(document.id, len(order))) return rows[:limit] @@ -160,7 +290,5 @@ def delete(db: DBSession, document: Document) -> None: path = stored_path(document.stored_name) if path is not None: path.unlink(missing_ok=True) - # Shares carry no foreign key to their resource, so nothing cascades. - sharing.forget_resource(db, document) db.delete(document) db.commit() diff --git a/src/lembas/services/sharing.py b/src/lembas/services/sharing.py index 9da65e4..e885063 100644 --- a/src/lembas/services/sharing.py +++ b/src/lembas/services/sharing.py @@ -3,6 +3,11 @@ One rule, in one place, for all three: you can see a resource if you own it, if it was shared with you by name, or if it was shared with a group you are in. +Documents are deliberately absent from that list. They are shared through the +knowledge base they belong to -- "this folder is the team's" is the granularity +people think in, and per-document grants would mean answering "who can see +this?" by checking every file. See services.library.documents.visible. + Everything that lists or searches a library store goes through `visible_to`. Writing the same condition into each query would work right up until one of them was written slightly differently, and the way that failure shows up is @@ -26,10 +31,10 @@ from sqlalchemy.orm import Session as DBSession from lembas.db.models import ( PRINCIPAL_GROUP, PRINCIPAL_USER, - RESOURCE_DOCUMENT, + RESOURCE_BASE, RESOURCE_NOTE, RESOURCE_SKILL, - Document, + KnowledgeBase, Note, Share, Skill, @@ -41,7 +46,7 @@ log = logging.getLogger(__name__) # The mapping between a model class and the string stored in Share. Kept here # so no caller has to remember which literal goes with which table. RESOURCE_TYPES: dict[Any, str] = { - Document: RESOURCE_DOCUMENT, + KnowledgeBase: RESOURCE_BASE, Note: RESOURCE_NOTE, Skill: RESOURCE_SKILL, } diff --git a/src/lembas/services/tools.py b/src/lembas/services/tools.py index 104ee38..f23a108 100644 --- a/src/lembas/services/tools.py +++ b/src/lembas/services/tools.py @@ -73,6 +73,9 @@ class ToolContext: owner_id: str search_config: dict[str, Any] = field(default_factory=dict) allow_private_fetch: bool = False + # Which knowledge bases this chat is scoped to. Empty means "everything the + # owner can see", which is what a chat with none attached should do. + base_ids: list[str] = field(default_factory=list) @dataclass @@ -171,7 +174,9 @@ async def _run_knowledge_search(context: ToolContext, args: dict[str, Any]) -> T with session_scope() as db: user = db.get(User, context.owner_id) - found = documents_service.search(db, user, query, limit=6) + found = documents_service.search( + db, user, query, limit=6, base_ids=context.base_ids + ) event = { "name": "knowledge_search", "query": query, @@ -677,11 +682,12 @@ def enabled_tools(db: DBSession, chat: Chat, user: User | None) -> list[dict[str return [tool.schema for tool in REGISTRY.values() if tool.family in families] -def context_for(db: DBSession, user: User | None) -> ToolContext: +def context_for(db: DBSession, user: User | None, chat: Chat | None = None) -> ToolContext: """The snapshot a running tool needs, taken while the session is open.""" return ToolContext( owner_id=user.id if user else "", search_config=settings_store.search(db), + base_ids=[base.id for base in chat.knowledge_bases] if chat is not None else [], ) diff --git a/src/lembas/web/static/css/app.css b/src/lembas/web/static/css/app.css index 06fd3d6..aeca9ed 100644 --- a/src/lembas/web/static/css/app.css +++ b/src/lembas/web/static/css/app.css @@ -231,6 +231,43 @@ button, input, textarea, select { box-shadow: 0 0 0 3px var(--accent-soft); } .input::placeholder, .textarea::placeholder { color: var(--ink-faint); } + +/* + File inputs. + + A file input is two things in one box -- a button the browser draws and the + chosen filename beside it -- and neither inherits anything useful. Left alone + with `.input`, the padding applies to the whole control so the button sits + hard against the left edge while the text floats off its centre line. + + So: no horizontal padding on the control, the button styled to the same + height as everything else and given the right border that separates it, and + the filename centred with line-height rather than flexbox, which file inputs + do not lay out reliably. +*/ +.input[type="file"] { + padding: 0 var(--control-px) 0 0; + line-height: calc(var(--control-h) - 2px); + cursor: pointer; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; + color: var(--ink-muted); +} +.input[type="file"]::file-selector-button { + height: calc(var(--control-h) - 2px); + margin: 0 var(--sp-3) 0 0; + padding: 0 var(--control-px); + border: 0; + border-right: 1px solid var(--border); + background: var(--surface-hover); + color: var(--ink); + font: inherit; + font-weight: 500; + cursor: pointer; + transition: background var(--transition-fast); +} +.input[type="file"]:hover::file-selector-button { background: var(--surface-active); } .input--mono, .textarea--mono { font-family: var(--font-mono); font-size: var(--text-xs); } .select { diff --git a/src/lembas/web/templates/chat/_knowledge_picker.html b/src/lembas/web/templates/chat/_knowledge_picker.html index a20e7e6..1c65fea 100644 --- a/src/lembas/web/templates/chat/_knowledge_picker.html +++ b/src/lembas/web/templates/chat/_knowledge_picker.html @@ -25,7 +25,7 @@ {{ document.title }} - {{ document.kind }} + {% if document.base %}{{ document.base.name }} · {% endif %}{{ document.kind }} {%- if document.pages %} · {{ document.pages }}p{% endif %} {%- if document.owner_id != user.id %} · shared{% endif %} diff --git a/src/lembas/web/templates/chat/index.html b/src/lembas/web/templates/chat/index.html index e87ec2d..c4dc057 100644 --- a/src/lembas/web/templates/chat/index.html +++ b/src/lembas/web/templates/chat/index.html @@ -68,6 +68,36 @@ {% endif %} + {% if knowledge_bases %} + {# Which bases this chat draws on. None ticked means everything you can + see, which is what a chat with nothing chosen should do. #} +
+ +
+ {# Always submitted, so unticking the last box still says something. + An absent checkbox carries no signal of its own. #} + +
+ {% for base in knowledge_bases %} + + {% endfor %} +
+
+

+ {% if attached_base_ids %} + Searches in this chat are limited to what is ticked. + {% else %} + Nothing ticked, so this chat can search everything in your + library. + {% endif %} +

+
+ {% endif %} + {% if can.get("chat.params") %}
diff --git a/src/lembas/web/templates/library/base_detail.html b/src/lembas/web/templates/library/base_detail.html new file mode 100644 index 0000000..c020087 --- /dev/null +++ b/src/lembas/web/templates/library/base_detail.html @@ -0,0 +1,126 @@ +{% extends "library/_layout.html" %} +{% from "_macros.html" import icon %} +{% set section = "knowledge" %} + +{% block title %}{{ base.name }} - LLeMbas{% endblock %} +{% block heading %}{{ base.name }}{% endblock %} + +{% block library_content %} +
+ + {{ icon("chevron-right", "icon--sm") }} All knowledge bases + +
+ +{% if is_owner %} +
+

Add to this base

+
+
+ +
+ + +

+ Images, PDFs and text. A PDF has its text read once, now. +

+
+ +
+ +
+ +
+ + +

+ Fetched now and kept as text, so it survives the page changing. +

+
+ +
+
+
+{% endif %} + +
+ + + {% if q %}Clear{% endif %} +
+ +{% if not documents %} +
+ {{ icon("archive", "empty__mark") }} +

{{ "Nothing found" if q else "Empty" }}

+

+ {% if q %}Nothing in this base matches “{{ q }}”. + {% else %}Add a file or a web page above.{% endif %} +

+
+{% else %} +
    + {% for document in documents %} +
  • +
    + + {{ document.title }} + +
    + {{ document.kind }} + {%- if document.pages %} · {{ document.pages }} page{{ '' if document.pages == 1 else 's' }}{% endif %} + {%- if document.size_bytes %} · {{ document.human_size }}{% endif %} + {%- if document.source_url %} · {{ document.source_url[:60] }}{% endif %} +
    + {% if document.description %} +
    {{ document.description }}
    + {% endif %} +
    +
    + {% if document.extraction_error %} + no text + {% endif %} +
    +
    • + {% endfor %} +
+{% include "library/_pager.html" %} +{% endif %} + +
+
+

This base

+
+
+ + +
+
+ + +
+
+
+ + {% include "library/_share.html" %} + + {% if is_owner %} +
+ + + +
+ {% endif %} +
+{% endblock %} diff --git a/src/lembas/web/templates/library/knowledge.html b/src/lembas/web/templates/library/knowledge.html index e5a76c2..0b48ddd 100644 --- a/src/lembas/web/templates/library/knowledge.html +++ b/src/lembas/web/templates/library/knowledge.html @@ -7,85 +7,60 @@ {% block library_content %}

- Documents, images and saved web pages you have collected. A model with the - knowledge tool searches these before it searches the web, and you can attach - any of them to a message. + Knowledge bases are collections of documents, images and saved web pages. Keep + them separate — one per subject, project or client — and a chat can be pointed + at just the ones it should draw on. Sharing happens here too: share a base and + everything in it comes with it.

-
-

Add

-
-
-
- - -

- Images, PDFs and text. PDFs have their text read once, now. -

-
- -
+{% if error %} +
{{ icon("warning", "alert__icon") }} {{ error }}
+{% endif %} -
-
- - -

- Fetched now and kept as text, so it survives the page changing. -

-
- -
-
-
- -
- - - {% if q %}Clear{% endif %} -
- -{% if not documents %} -
- {{ icon("archive", "empty__mark") }} -

{{ "Nothing found" if q else "The shelves are bare" }}

-

- {% if q %} - No document matches “{{ q }}”. - {% else %} - Add a file or a web page above and it becomes searchable — by you, and by - any model you have given the knowledge tool. - {% endif %} -

-
-{% else %} +{% if bases %}
    - {% for document in documents %} + {% for base in bases %}
  • - {{ document.title }} + {{ base.name }}
    - {{ document.kind }} - {%- if document.pages %} · {{ document.pages }} page{{ '' if document.pages == 1 else 's' }}{% endif %} - {%- if document.size_bytes %} · {{ document.human_size }}{% endif %} - {%- if document.source_url %} · {{ document.source_url[:60] }}{% endif %} + {{ counts.get(base.id, 0) }} document{{ '' if counts.get(base.id, 0) == 1 else 's' }} + {%- if base.description %} · {{ base.description }}{% endif %}
    - {% if document.description %} -
    {{ document.description }}
    - {% endif %}
    - {% if document.extraction_error %} - no text - {% endif %} - {% if document.owner_id != user.id %}shared{% endif %} + {% if base.owner_id != user.id %}shared with you{% endif %}
    • {% endfor %}
-{% include "library/_pager.html" %} +{% else %} +
+ {{ icon("archive", "empty__mark") }} +

No knowledge bases yet

+

+ Make one below, then put documents in it. A chat with no base attached + searches everything you have; a chat pointed at one searches only that. +

+
{% endif %} + +
+

New knowledge base

+
+
+
+ + +
+
+ + +
+
+ +
+
{% endblock %} diff --git a/src/lembas/web/templates/library/knowledge_detail.html b/src/lembas/web/templates/library/knowledge_detail.html index 285593f..ed480da 100644 --- a/src/lembas/web/templates/library/knowledge_detail.html +++ b/src/lembas/web/templates/library/knowledge_detail.html @@ -7,7 +7,9 @@ {% block library_content %}
- {{ icon("chevron-right", "icon--sm") }} All documents + + {{ icon("chevron-right", "icon--sm") }} Back to {{ document.base.name if document.base else "the base" }} +
@@ -28,6 +30,22 @@

+ {% if document.base %} +
+ + {# Moving a document changes who can see it, which is the point of bases. + Only bases this person can write to are offered. #} + +

Moving it changes who can see it.

+
+ {% endif %} +
Kind
{{ document.kind }}
{% if document.source_url %} @@ -50,8 +68,6 @@ {% endif %} - {% include "library/_share.html" %} - {% if is_owner %}
diff --git a/tests/test_sharing.py b/tests/test_sharing.py index 1ea6ffd..2142814 100644 --- a/tests/test_sharing.py +++ b/tests/test_sharing.py @@ -12,7 +12,6 @@ from sqlalchemy import select from lembas.db.models import ( PRINCIPAL_GROUP, PRINCIPAL_USER, - Document, Group, Note, Share, @@ -147,25 +146,25 @@ def test_forgetting_a_principal_drops_their_shares(db, people): def test_two_kinds_of_resource_do_not_collide(db, people): """One shares table across three resource types, so the type must be part - of the match -- otherwise a note and a document sharing an id would share - each other's access.""" + of the match -- otherwise a note and a base sharing an id would share each + other's access.""" note = _note(db, people["frodo"]) - document = documents_service.store_upload( - db, owner=people["frodo"], payload=b"hello", filename="a.txt" - ) + base = documents_service.create_base(db, owner=people["frodo"], name="Papers") sharing.set_grants(db, note, user_ids=[people["gollum"].id], group_ids=[]) assert sharing.can_read(db, note, people["gollum"]) - assert not sharing.can_read(db, document, people["gollum"]) + assert not sharing.can_read(db, base, people["gollum"]) -def test_resource_type_refuses_something_unshareable(db, people): - """Memory is deliberately not shareable: a record about a person is not - content to hand round.""" - from lembas.db.models import Memory +@pytest.mark.parametrize("unshareable", ["Memory", "Document"]) +def test_resource_type_refuses_something_unshareable(db, people, unshareable): + """Memory is not shareable at all -- a record about a person is not content + to hand round. A Document is shared through the base it lives in, so asking + to share one directly is a mistake worth catching loudly.""" + import lembas.db.models as models with pytest.raises(ValueError): - sharing.resource_type(Memory) + sharing.resource_type(getattr(models, unshareable)) # --- Through the search path ------------------------------------------------- @@ -180,20 +179,53 @@ def test_search_does_not_leak_across_owners(db, people): assert notes_service.search(db, people["gandalf"], "golden") == [] -def test_a_shared_document_is_findable_by_the_person_it_was_shared_with(db, people): +def test_sharing_a_base_shares_what_is_in_it(db, people): + """Documents are shared through their base. "This folder is the team's" is + the granularity people think in, and per-document grants would mean + answering "who can see this?" by checking every file.""" + base = documents_service.create_base(db, owner=people["frodo"], name="Trees") document = documents_service.store_upload( db, owner=people["frodo"], payload=b"The mallorn is a golden tree.", filename="tree.txt", + base=base, ) assert documents_service.search(db, people["gollum"], "mallorn") == [] - sharing.set_grants(db, document, user_ids=[people["gollum"].id], group_ids=[]) + sharing.set_grants(db, base, user_ids=[people["gollum"].id], group_ids=[]) found = documents_service.search(db, people["gollum"], "mallorn") assert [d.id for d in found] == [document.id] +def test_a_document_in_an_unshared_base_stays_private(db, people): + """Two bases, one shared: the other must not come with it.""" + shared = documents_service.create_base(db, owner=people["frodo"], name="Public") + private = documents_service.create_base(db, owner=people["frodo"], name="Private") + documents_service.store_upload( + db, owner=people["frodo"], payload=b"A mallorn tree.", filename="a.txt", base=shared + ) + documents_service.store_upload( + db, owner=people["frodo"], payload=b"A mallorn secret.", filename="b.txt", base=private + ) + sharing.set_grants(db, shared, user_ids=[people["gollum"].id], group_ids=[]) + + found = documents_service.search(db, people["gollum"], "mallorn") + assert [d.base_id for d in found] == [shared.id] + + +def test_scoping_to_a_base_cannot_be_used_to_reach_one(db, people): + """Naming a base you cannot see returns nothing rather than granting it.""" + private = documents_service.create_base(db, owner=people["frodo"], name="Private") + documents_service.store_upload( + db, owner=people["frodo"], payload=b"A mallorn tree.", filename="a.txt", base=private + ) + found = documents_service.search( + db, people["gollum"], "mallorn", base_ids=[private.id] + ) + assert found == [] + + def test_the_shares_table_records_what_was_asked_for(db, people): group = Group(name="Fellowship") db.add(group) @@ -222,10 +254,33 @@ def test_visibility_is_a_query_filter_not_a_python_loop(db, people): assert [n.title for n in rows] == ["Note 0", "Note 1"] -def test_documents_and_notes_use_the_same_rule(db, people): +def test_a_document_follows_its_base(db, people): document = documents_service.store_upload( db, owner=people["frodo"], payload=b"x", filename="a.txt" ) assert document in db.scalars(documents_service.visible(db, people["frodo"])) assert document not in db.scalars(documents_service.visible(db, people["gollum"])) - assert list(db.scalars(select(Document).where(sharing.visible_to(Document, None)))) == [] + assert list(db.scalars(documents_service.visible(db, None))) == [] + + +def test_an_uploaded_document_always_lands_in_a_base(db, people): + """base_id is nullable only so the column could be added to a table that + already had rows; the service never leaves it unset.""" + document = documents_service.store_upload( + db, owner=people["frodo"], payload=b"x", filename="a.txt" + ) + assert document.base_id is not None + + +def test_documents_predating_bases_are_filed_at_startup(db, people): + document = documents_service.store_upload( + db, owner=people["frodo"], payload=b"x", filename="a.txt" + ) + document.base_id = None + db.commit() + assert document not in db.scalars(documents_service.visible(db, people["frodo"])) + + assert documents_service.sweep_unfiled(db) == 1 + db.refresh(document) + assert document.base_id is not None + assert document in db.scalars(documents_service.visible(db, people["frodo"])) diff --git a/tests/test_tools.py b/tests/test_tools.py index 2dc1905..c53b2e4 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -166,6 +166,48 @@ def _context(**kwargs): return tools_service.ToolContext(owner_id="someone", **kwargs) +# --- Knowledge is scoped to the chat's bases --------------------------------- +async def test_knowledge_search_is_limited_to_the_attached_bases(db, user_id): + """"Answer from the contracts folder" is a different question from "answer + from everything I have ever uploaded".""" + from lembas.db.models import User + from lembas.services.library import documents as documents_service + + owner = db.get(User, user_id) + trees = documents_service.create_base(db, owner=owner, name="Trees") + contracts = documents_service.create_base(db, owner=owner, name="Contracts") + documents_service.store_upload( + db, owner=owner, payload=b"The mallorn is golden.", filename="a.txt", + title="Mallorn", base=trees, + ) + documents_service.store_upload( + db, owner=owner, payload=b"The mallorn clause is void.", filename="b.txt", + title="Clause", base=contracts, + ) + + everywhere = await tools_service.run_tool( + tools_service.ToolContext(owner_id=user_id), "knowledge_search", + '{"query": "mallorn"}', + ) + assert {r["title"] for r in everywhere.event["results"]} == {"Mallorn", "Clause"} + + scoped = await tools_service.run_tool( + tools_service.ToolContext(owner_id=user_id, base_ids=[contracts.id]), + "knowledge_search", + '{"query": "mallorn"}', + ) + assert [r["title"] for r in scoped.event["results"]] == ["Clause"] + + +def test_a_chat_with_no_bases_searches_everything(db, user_id): + """Empty means "everything the owner can see", not "nothing".""" + from lembas.db.models import User + + chat = _chat_with(db, user_id, capabilities={"tools": True}) + context = tools_service.context_for(db, db.get(User, user_id), chat) + assert context.base_ids == [] + + # --- Running one ------------------------------------------------------------- async def test_running_web_search_formats_results_for_the_model(monkeypatch): async def fake_run(_config, query, *, limit=None):