diff --git a/CLAUDE.md b/CLAUDE.md index fd0aa51..2bfbf2e 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -19,7 +19,7 @@ lembas info # paths + counts, useful when confused lembas secret-key # generate LEMBAS_SECRET_KEY lembas create-admin # create or promote an admin -pytest # 338 tests, ~16s +pytest # 428 tests, ~26s # PLAN.md tracks what is and is not built ruff check . # lint (line length 100) python scripts/build_artwork.py # regenerate artwork (SVG + PWA icons; @@ -81,6 +81,7 @@ src/lembas/ admin_audio.py speech-to-text and text-to-speech endpoints admin_search.py web search provider and credentials audio.py transcribe, speak, voice discovery + library.py knowledge, notes, skills pages; memory CRUD files.py upload, serve, remove attachments preferences.py per-user theme, default model, password, audio db/ @@ -92,7 +93,11 @@ src/lembas/ services/ llm/openai_client.py httpx streaming + model discovery search/ ddgs, SearXNG and Firecrawl behind one shape + library/ documents, notes, memories, skills, FTS audio.py OpenAI-shaped /v1/audio/* client + fetch.py URL retrieval, HTML to text, the SSRF guard + sharing.py one visibility rule for every library store + harness.py the operational prompt built from what a model has tools.py tool registry, schemas, streamed-call reassembly chat.py request building, endpoint resolution, titles markdown.py markdown-it + pygments + nh3 @@ -293,6 +298,57 @@ do without `vision`. chunks. `tools.ToolCallAccumulator` rejoins them keyed on `index` — not on name, which breaks the moment a model calls one tool twice in a turn. +**Four stores, four different reasons.** `services/library/` — `documents` +(uploaded by a person, searched by the model), `notes` (written by the model, +searched), `memories` (short, and *injected whole* every turn), `skills` (index +injected, body fetched by tool). The shape of each follows from how it reaches +the model: a memory is capped short because it costs tokens on every request +forever, a note is not injected because a dozen would fill the window. + +**Sharing goes through one helper, and admins do not bypass it.** +`services/sharing.py:visible_to()` is the only definition of who can see a +library item, and every listing and tool uses it. `permissions.resolve` gives an +admin everything, deliberately — but that is about configuration, which an admin +can grant themselves anyway. Reading someone's private notes is not the same +act, so `sharing` has no admin branch. Sharing grants **reading only**. + +**FTS5 tables are outside the model-driven schema sync.** They are not +SQLAlchemy models, so `sync_schema()` cannot diff them; `db/migrations.py: +ensure_fts()` writes them out with `IF NOT EXISTS` and creates the triggers that +keep an external-content index correct. It runs at every startup and converges, +like the column sync beside it. `tests/conftest.py` calls `sync_schema` rather +than `create_all` so tests run against the same schema. + +**A failed search rolls back.** One broken FTS statement otherwise leaves the +session unusable and every later query in the request fails too, which looks +nothing like a search problem. + +**Knowledge attachments are copies.** Attaching a library document to a message +duplicates its text and its file (`files.copy_document`). Referencing it would +mean a conversation changing when a document is edited or deleted later — the +same reason PDF text is extracted once at upload. + +**The link fetcher is an SSRF hole unless guarded.** `services/fetch.py` refuses +loopback, private and link-local addresses **after resolution** — a hostname +pointing at 127.0.0.1 walks past any check that only reads the URL — and follows +redirects by hand so every hop is checked. An admin can open it deliberately. +The URL can come from a model, which can be talked into things by a page it just +read. + +**The harness is an exception to the prompt-precedence rule, on purpose.** +"System prompts are precedence, not concatenation" governs the three *authored* +layers, and it stands: exactly one still wins, and `effective_system_prompt` +still decides which. `services/harness.py` is a different axis — it describes +the machinery rather than the behaviour, nobody authored it, and there is +nothing for it to disagree with. It is prepended to whichever authored prompt +won, in one system message (several endpoints reject a second one), and +`build_request` is where the two meet. + +**A model's tool flags default to on when `tools` is on.** Rows configured +before the per-tool split have no `tool_*` keys. Reading absent as off would +silently take web search away from every model already set up for it, so +`tools.enabled_tools` treats absent as inherited. + **Tool results are not replayed.** Like reasoning, `Message.tool_calls_json` is stored and rendered but never fed back as context. The answer already contains what the model made of the results; replaying stale results and the schema into @@ -368,8 +424,9 @@ notes describe the machine. Custom tools and MCP, agentic execution (local subprocess and SSH connection profiles), image generation. Nav entries mark where each one goes. The tool -loop in `services/generation.py` is what they plug into — a second tool is a -registry entry, not a new code path. +loop in `services/generation.py` is what they plug into — a new tool is a +`ToolDef` in `services/tools.py:REGISTRY` plus a permission and a capability +flag, not a new code path. No OCR: a scanned PDF is stored with an explanatory `extraction_error` rather than silently contributing nothing. diff --git a/PLAN.md b/PLAN.md index e0e8db7..3e71e56 100644 --- a/PLAN.md +++ b/PLAN.md @@ -5,8 +5,9 @@ that would be expensive to revisit. Kept current as work lands; the detail of *how* things work lives in [`CLAUDE.md`](CLAUDE.md). **Status:** usable daily. Streaming chat, attachments, reasoning, tool calling -with web search, speech in and out, users and groups, model administration, -installable as an app. 338 tests, `ruff` clean. +with web search, a knowledge library, notes, memory and skills, speech in and +out, users and groups, model administration, installable as an app. 428 tests, +`ruff` clean. --- @@ -57,6 +58,25 @@ be a different project, not a refactor. - [x] Sources stay in the transcript; results are **not** replayed as context on the next turn, for the same reasons reasoning is not +### The library +- [x] **Knowledge** — documents, images and saved web pages, ingested through + the same pipeline as chat attachments, searched with SQLite FTS5 +- [x] **Notes** — longer things the model writes down and searches later; + editable by hand, because they are yours +- [x] **Memory** — short facts, injected on every turn to a budget rather than + searched, and managed in your settings +- [x] **Skills** — saved procedures. Only the name and description are injected; + the body is fetched when the model decides it applies +- [x] A model may write and revise its own notes, memories and skills. Every + skill revision is kept, attributed and revertible — the safety story is a + record and a way back, not a gate +- [x] **Sharing** — any of the three can be shared with a group or with named + people, read-only. One visibility rule, and administrators do not bypass it +- [x] **The harness** — an operational prompt assembled from what a model + actually has, so the tools get used rather than ignored +- [x] Attach menu: file, image, a web page fetched on the spot, or a document + from the library + ### Audio - [x] **Dictation** — record in the composer, transcribed by any OpenAI-shaped `/v1/audio/transcriptions` endpoint. The recording never touches disk @@ -114,10 +134,11 @@ be a different project, not a refactor. In the order they are likely to be worth doing. ### Custom tools and MCP servers -An MCP client managing configured servers, their tools surfaced alongside -`web_search`. The loop they plug into exists now — `services/tools.py` is a -registry and `services/generation.py` already runs bounded rounds — so this is -a client and an admin screen rather than a change to how chat works. +An MCP client managing configured servers, their tools surfaced alongside the +built-in ones. The loop they plug into exists now — `services/tools.py` is a +registry of thirteen tools and `services/generation.py` already runs bounded +rounds — so this is a client and an admin screen rather than a change to how +chat works. ### Agentic execution Two modes, as originally specified: @@ -137,6 +158,8 @@ already running on this machine and is the obvious first target. - **Conversation branching** — `Message.parent_id` exists unused; needs a UI for choosing between versions, which is why rewind truncates for now - **Chat export** (Markdown, JSON) +- **Semantic search** in the library — the retrieval service is one call, so an + embedding backend can go behind it without touching the tools or the UI - **Archived chats** — the column exists, nothing surfaces it - **Per-user quotas** @@ -172,6 +195,15 @@ microphone is unavailable for the same reason. administrator's assertion, not something endpoints reliably advertise. Set it on a model that cannot, and its replies fail rather than degrade. +**Library search is keyword, not semantic.** FTS5 ranks well and needs no +dependency or embedding endpoint, but "how do I get paid" will not find a +document that says "invoicing". + +**A model can write its own skills, and they take effect at once.** Marked as +model-authored and fully revertible, but a model that has just read a hostile +page could save a skill that outlives the conversation. The mitigation is that +it is visible and undoable, not that it was prevented. + --- ## Deliberate decisions @@ -191,6 +223,15 @@ Recorded because each looks like an oversight until you know the reason. - **Images only reach models marked `vision`.** Not graceful degradation: most endpoints reject the entire request rather than ignoring an image part. Tools are gated the same way, for the same reason. +- **Sharing grants reading, never writing.** Two people editing one note with no + history and no merge is worse than the inconvenience of copying it. +- **Memory is never shareable.** A record about a person is not content to hand + round. +- **Knowledge attached to a message is copied, not referenced.** History must not + change under a conversation because a document was edited later. +- **The harness is prepended to the authored prompt, not a fourth layer.** It + describes the machinery; the authored layers describe the behaviour. Only one + authored layer still wins. - **Tool results are not replayed.** Like reasoning: the answer already contains what the model made of them, and replaying stale results into every later request wastes the window and sends small models into search loops. diff --git a/README.md b/README.md index cb1158a..af465fd 100644 --- a/README.md +++ b/README.md @@ -54,6 +54,12 @@ runtime. Clone it, `pip install -e .`, run it. - **Speech in and out** — dictate a message and have replies read aloud, against any OpenAI-compatible audio endpoint (whisper.cpp, Speaches, Kokoro…). Each person picks their own voice +- **A library** — four places a model can reach for. **Knowledge**: documents, + images and web pages you collect, searched before the web. **Notes**: longer + things it writes down and finds again later. **Memory**: short facts about you, + in front of it on every turn. **Skills**: saved procedures it can follow, and + write. All of it visible and editable by you, and shareable with a group or a + person, read-only - **Installable** — add it to a phone home screen or a desktop launcher and it runs in its own window - **OpenAI connections** — point at OpenAI, LM Studio, vLLM, llama.cpp, @@ -73,7 +79,7 @@ runtime. Clone it, `pip install -e .`, run it. **Planned** Custom tools and MCP servers · agentic execution (local and over SSH) · image -generation · OCR for scanned PDFs. +generation · OCR for scanned PDFs · semantic search in the library. See [PLAN.md](PLAN.md) for what is built, what is not, and why. @@ -130,6 +136,22 @@ Recorded audio is passed straight through and never written to disk. > The microphone needs HTTPS or localhost. Browsers do not grant it over plain > HTTP, so a LAN install without TLS will not offer dictation. +### The library + +**Sidebar → Library**, and **Settings → Memory**. Nothing is on by default for a +model: give it the tools it should have under **Admin → Models**, where +`tools` decides whether a tool list may be sent at all and the built-in tools are +chosen one by one. + +Search is SQLite's FTS5 — keyword matching with BM25 ranking, no embedding +service to run and nothing that stops working offline. It will not match a +paraphrase, so a line of description on a document is worth writing. + +> Saving a **link** makes your server fetch a URL. Addresses on your own machine +> and network are refused unless an administrator opts in under +> **Admin → Web search**, because the address can come from a model and the +> server can reach things your browser cannot. + ### Installing as an app Open it in a browser and use *Install* (Chromium) or *Share → Add to Home diff --git a/src/lembas/api/admin_models.py b/src/lembas/api/admin_models.py index 9814ba4..813b488 100644 --- a/src/lembas/api/admin_models.py +++ b/src/lembas/api/admin_models.py @@ -18,7 +18,22 @@ log = logging.getLogger(__name__) router = APIRouter(tags=["admin-models"]) -CAPABILITIES = ("reasoning", "vision", "tools") +# What the endpoint can do. Endpoints do not advertise any of this reliably, so +# these are an administrator's assertion. +PROTOCOL_CAPABILITIES = ("reasoning", "vision", "tools") + +# Which built-in tools this model is given. Distinct from the above: `tools` is +# whether a tools array may be sent at all, these are what goes in it. Every one +# of them is meaningless unless `tools` is on. +TOOL_CAPABILITIES = ( + ("tool_web_search", "Web search"), + ("tool_knowledge", "Knowledge"), + ("tool_notes", "Notes"), + ("tool_memory", "Memory"), + ("tool_skills", "Skills"), +) + +CAPABILITIES = PROTOCOL_CAPABILITIES + tuple(key for key, _ in TOOL_CAPABILITIES) def _model(db: DBSession, model_id: str) -> Model: @@ -138,7 +153,13 @@ async def model_detail( { "model": model, "groups": list(db.scalars(select(Group).order_by(Group.name))), - "capabilities": CAPABILITIES, + "capabilities": PROTOCOL_CAPABILITIES, + "tool_capabilities": TOOL_CAPABILITIES, + # Rows predating the split have no tool_* keys at all. Showing them + # unticked would be a lie: tools.enabled_tools treats absent as on + # when `tools` is on, so that an upgrade does not silently take web + # search away from every model already configured for it. + "tool_default": bool((model.capabilities_json or {}).get("tools")), "default_model": settings_store.get(db, "default_model") or "", "instance_prompt": settings_store.get(db, "system_prompt") or "", "position_of": index + 1, diff --git a/src/lembas/api/admin_search.py b/src/lembas/api/admin_search.py index 251148a..c6f6c44 100644 --- a/src/lembas/api/admin_search.py +++ b/src/lembas/api/admin_search.py @@ -56,6 +56,7 @@ async def save_search( firecrawl_base_url: str = Form(""), firecrawl_api_key: str = Form(""), timeout: float = Form(20.0), + allow_private_fetch: bool = Form(False), ) -> Response: current = settings_store.search(db) known = {p.key for p in search_service.PROVIDERS} @@ -77,6 +78,7 @@ async def save_search( firecrawl_api_key, current.get("firecrawl_api_key_encrypted") or "" ), "timeout": min(max(timeout, 5.0), 120.0), + "allow_private_fetch": allow_private_fetch, }, key=settings_store.SEARCH, ) diff --git a/src/lembas/api/files.py b/src/lembas/api/files.py index 5d6e22f..0303642 100644 --- a/src/lembas/api/files.py +++ b/src/lembas/api/files.py @@ -4,12 +4,25 @@ from __future__ import annotations import logging -from fastapi import APIRouter, Depends, File, HTTPException, Request, Response, UploadFile, status +from fastapi import ( + APIRouter, + Depends, + File, + Form, + HTTPException, + Request, + Response, + UploadFile, + status, +) from fastapi.responses import FileResponse from lembas.api.deps import Db, RequiredUser, require_permission -from lembas.db.models import Attachment +from lembas.db.models import Attachment, Document from lembas.services import files as files_service +from lembas.services import settings_store +from lembas.services.fetch import FetchError, fetch +from lembas.services.library import documents as documents_service from lembas.web.templating import templates log = logging.getLogger(__name__) @@ -65,6 +78,95 @@ async def upload( ) +@router.post("/link", dependencies=[Depends(require_permission("files.upload"))]) +async def attach_link( + request: Request, db: Db, user: RequiredUser, url: str = Form(""), chat_id: str = Form("") +) -> Response: + """Fetch a web page and attach its text. + + The page is reduced to text here and stored, rather than being fetched again + when the message is sent: the same rule as PDF extraction. A reply must not + change because a page was edited between composing and sending. + """ + config = settings_store.search(db) + try: + page = await fetch(url, allow_private=bool(config.get("allow_private_fetch"))) + except FetchError as exc: + return templates.TemplateResponse( + request, + "chat/_attachment_error.html", + {"request": request, "filename": url[:80] or "link", "error": exc.message}, + ) + + attachment = files_service.store_text( + db, + user_id=user.id, + chat_id=chat_id or None, + filename=f"{page.title[:120] or 'page'}.txt", + text=page.text, + truncated=page.truncated, + source_note=page.url, + ) + return templates.TemplateResponse( + request, "chat/_attachment_chip.html", {"request": request, "attachment": attachment} + ) + + +@router.post("/from-knowledge", dependencies=[Depends(require_permission("files.upload"))]) +async def attach_from_knowledge( + request: Request, db: Db, user: RequiredUser, document_id: str = Form(""), + chat_id: str = Form(""), +) -> Response: + """Attach a library document to the message being composed. + + The document is **copied**, not referenced. History must not change under a + conversation because a document was later edited or deleted -- the same + reason a PDF's text is extracted once at upload rather than per request. + """ + document = documents_service.get(db, document_id, user) + if document is None: + return templates.TemplateResponse( + request, + "chat/_attachment_error.html", + { + "request": request, + "filename": "document", + "error": "That document is not available.", + }, + ) + + attachment = files_service.copy_document( + db, user_id=user.id, chat_id=chat_id or None, document=document + ) + return templates.TemplateResponse( + request, "chat/_attachment_chip.html", {"request": request, "attachment": attachment} + ) + + +@router.get("/knowledge-picker", dependencies=[Depends(require_permission("files.upload"))]) +async def knowledge_picker( + request: Request, db: Db, user: RequiredUser, q: str = "", chat_id: str = "" +) -> Response: + """The list of documents shown by the composer's Knowledge option.""" + if q.strip(): + found = documents_service.search(db, user, q, limit=20) + else: + found = list( + db.scalars( + documents_service.visible(db, user) + .order_by(Document.created_at.desc()) + .limit(20) + ) + ) + return templates.TemplateResponse( + request, + "chat/_knowledge_picker.html", + # `user` is read by the template to mark documents shared by someone + # else; render() would inject it, but this is a fragment. + {"request": request, "documents": found, "q": q, "chat_id": chat_id, "user": user}, + ) + + @router.delete("/{attachment_id}") async def remove(db: Db, user: RequiredUser, attachment_id: str) -> Response: """Detach a file before it has been sent.""" diff --git a/src/lembas/api/library.py b/src/lembas/api/library.py new file mode 100644 index 0000000..e8f37fd --- /dev/null +++ b/src/lembas/api/library.py @@ -0,0 +1,450 @@ +"""The library: knowledge documents, notes, skills — and memory in settings. + +List-plus-detail throughout, the same shape as the model admin: compact rows +with search and pagination, and a full form on its own page. A library is +expected to run to hundreds of items, and a page that renders a form per row is +unusable at that size. + +Every read goes through ``services.sharing.visible_to`` and every write through +``owner_id``. Sharing grants reading only -- two people editing one note with no +history and no merge is worse than the inconvenience of copying it. +""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, File, Form, HTTPException, Request, UploadFile, status +from fastapi.responses import FileResponse, RedirectResponse, Response +from sqlalchemy import func, select +from sqlalchemy.orm import Session as DBSession + +from lembas.api.deps import Db, RequiredUser, require_permission +from lembas.api.pages import sidebar_context +from lembas.db.models import ( + AUTHOR_USER, + PRINCIPAL_GROUP, + PRINCIPAL_USER, + Document, + Group, + Note, + Skill, + SkillRevision, + User, +) +from lembas.security import permissions +from lembas.services import files as files_service +from lembas.services import settings_store, sharing +from lembas.services.fetch import FetchError, fetch +from lembas.services.library import documents as documents_service +from lembas.services.library import memories as memories_service +from lembas.services.library import notes as notes_service +from lembas.services.library import skills as skills_service +from lembas.services.markdown import render_markdown +from lembas.web.templating import render + +log = logging.getLogger(__name__) + +router = APIRouter(dependencies=[Depends(require_permission("library.use"))], tags=["library"]) + +PAGE_SIZE = 30 + + +def _page(db: DBSession, query, page: int): + """One page of a visibility-filtered query, plus what the pager needs.""" + total = db.scalar(select(func.count()).select_from(query.subquery())) or 0 + pages = max(1, (total + PAGE_SIZE - 1) // PAGE_SIZE) + page = min(max(page, 1), pages) + rows = list(db.scalars(query.offset((page - 1) * PAGE_SIZE).limit(PAGE_SIZE))) + return rows, {"page": page, "pages": pages, "total": total} + + +def _shared_context(db: DBSession, user: User, resource) -> dict: + """Everything the share panel on a detail page needs.""" + grants = sharing.grants_for(db, resource) + return { + "can_share": permissions.has(db, user, "library.share"), + "groups": list(db.scalars(select(Group).order_by(Group.name))), + "people": list( + db.scalars(select(User).where(User.id != user.id).order_by(User.name)) + ), + "shared_users": [g.principal_id for g in grants if g.principal_type == PRINCIPAL_USER], + "shared_groups": [g.principal_id for g in grants if g.principal_type == PRINCIPAL_GROUP], + "is_owner": resource.owner_id == user.id, + } + + +def _apply_shares(db: DBSession, user: User, resource, form) -> None: + if not permissions.has(db, user, "library.share") or resource.owner_id != user.id: + return + sharing.set_grants( + db, + resource, + user_ids=form.getlist("share_user"), + group_ids=form.getlist("share_group"), + ) + + +# --- Shell ------------------------------------------------------------------- +@router.get("/library") +async def library_home(user: RequiredUser): + return RedirectResponse("/library/knowledge", status_code=status.HTTP_303_SEE_OTHER) + + +# --- Knowledge --------------------------------------------------------------- +@router.get("/library/knowledge") +async def knowledge_list( + request: Request, db: Db, user: RequiredUser, q: str = "", page: int = 1, saved: str = "" +): + if q.strip(): + # Search returns best-match order and its own limit, so it is not paged. + rows = documents_service.search(db, user, q, limit=PAGE_SIZE) + pager = {"page": 1, "pages": 1, "total": len(rows)} + else: + rows, pager = _page( + db, documents_service.visible(db, user).order_by(Document.created_at.desc()), page + ) + return render( + request, + "library/knowledge.html", + { + "section": "knowledge", + "documents": rows, + "q": q, + "pager": pager, + "saved": saved, + **sidebar_context(db, user), + }, + ) + + +@router.get("/library/knowledge/{document_id}") +async def knowledge_detail(request: Request, db: Db, user: RequiredUser, document_id: str): + document = documents_service.get(db, document_id, user) + if document is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That document is not available.") + return render( + request, + "library/knowledge_detail.html", + { + "section": "knowledge", + "document": document, + **_shared_context(db, user, document), + **sidebar_context(db, user), + }, + ) + + +@router.post("/api/library/documents") +async def upload_document( + db: Db, user: RequiredUser, file: UploadFile = File(...), title: str = Form("") +) -> Response: + payload = await file.read(files_service.MAX_UPLOAD_BYTES + 1) + try: + document = documents_service.store_upload( + db, owner=user, payload=payload, filename=file.filename or "file", title=title + ) + except files_service.FileError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc + return RedirectResponse( + f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER + ) + + +@router.post("/api/library/documents/link") +async def save_link(db: Db, user: RequiredUser, url: str = Form(...)) -> Response: + config = settings_store.search(db) + try: + page = await fetch(url, allow_private=bool(config.get("allow_private_fetch"))) + except FetchError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, exc.message) from exc + document = documents_service.store_page(db, owner=user, page=page) + return RedirectResponse( + f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER + ) + + +@router.post("/api/library/documents/{document_id}") +async def update_document( + request: Request, db: Db, user: RequiredUser, document_id: str +) -> Response: + document = documents_service.get(db, document_id, user) + if document is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That document is not available.") + if not sharing.can_write(document, user): + raise HTTPException(status.HTTP_403_FORBIDDEN, "That document is not yours to change.") + + form = await request.form() + document.title = str(form.get("title", document.title)).strip()[:300] or document.title + document.description = str(form.get("description", "")).strip()[:2000] + db.commit() + _apply_shares(db, user, document, form) + return RedirectResponse( + f"/library/knowledge/{document.id}", status_code=status.HTTP_303_SEE_OTHER + ) + + +@router.post("/api/library/documents/{document_id}/delete") +async def delete_document(db: Db, user: RequiredUser, document_id: str) -> Response: + document = documents_service.get(db, document_id, user) + if document is None or not sharing.can_write(document, user): + raise HTTPException(status.HTTP_404_NOT_FOUND, "That document is not available.") + documents_service.delete(db, document) + return RedirectResponse("/library/knowledge", status_code=status.HTTP_303_SEE_OTHER) + + +@router.get("/api/library/documents/{document_id}/content") +async def document_content(db: Db, user: RequiredUser, document_id: str) -> Response: + """Serve a document's file. + + Non-images go out as attachments with nosniff, exactly as chat attachments + do: an uploaded .html must not be able to execute in this origin. + """ + document = documents_service.get(db, document_id, user) + if document is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That document is not available.") + path = documents_service.stored_path(document.stored_name) + if path is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That file is no longer on disk.") + + headers = {"X-Content-Type-Options": "nosniff"} + if not document.is_image: + headers["Content-Disposition"] = f'attachment; filename="{document.filename}"' + return FileResponse(path, media_type=document.media_type, headers=headers) + + +# --- Notes ------------------------------------------------------------------- +@router.get("/library/notes") +async def notes_list(request: Request, db: Db, user: RequiredUser, q: str = "", page: int = 1): + if q.strip(): + rows = notes_service.search(db, user, q, limit=PAGE_SIZE) + pager = {"page": 1, "pages": 1, "total": len(rows)} + else: + rows, pager = _page( + db, notes_service.visible(db, user).order_by(Note.updated_at.desc()), page + ) + return render( + request, + "library/notes.html", + { + "section": "notes", + "notes": rows, + "q": q, + "pager": pager, + **sidebar_context(db, user), + }, + ) + + +@router.get("/library/notes/new") +async def new_note(request: Request, db: Db, user: RequiredUser): + return render( + request, + "library/note_detail.html", + {"section": "notes", "note": None, **sidebar_context(db, user)}, + ) + + +@router.get("/library/notes/{note_id}") +async def note_detail(request: Request, db: Db, user: RequiredUser, note_id: str): + note = notes_service.get(db, note_id, user) + if note is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That note is not available.") + return render( + request, + "library/note_detail.html", + { + "section": "notes", + "note": note, + "body_html": render_markdown(note.body), + **_shared_context(db, user, note), + **sidebar_context(db, user), + }, + ) + + +@router.post("/api/library/notes") +async def create_note( + db: Db, user: RequiredUser, title: str = Form(""), body: str = Form("") +) -> Response: + note = notes_service.create(db, owner=user, title=title, body=body, author=AUTHOR_USER) + return RedirectResponse(f"/library/notes/{note.id}", status_code=status.HTTP_303_SEE_OTHER) + + +@router.post("/api/library/notes/{note_id}") +async def update_note(request: Request, db: Db, user: RequiredUser, note_id: str) -> Response: + note = notes_service.get(db, note_id, user) + if note is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That note is not available.") + if not sharing.can_write(note, user): + raise HTTPException(status.HTTP_403_FORBIDDEN, "That note is not yours to change.") + + form = await request.form() + notes_service.update(db, note, title=str(form.get("title", "")), body=str(form.get("body", ""))) + _apply_shares(db, user, note, form) + return RedirectResponse(f"/library/notes/{note.id}", status_code=status.HTTP_303_SEE_OTHER) + + +@router.post("/api/library/notes/{note_id}/delete") +async def delete_note(db: Db, user: RequiredUser, note_id: str) -> Response: + note = notes_service.get(db, note_id, user) + if note is None or not sharing.can_write(note, user): + raise HTTPException(status.HTTP_404_NOT_FOUND, "That note is not available.") + notes_service.delete(db, note) + return RedirectResponse("/library/notes", status_code=status.HTTP_303_SEE_OTHER) + + +# --- Skills ------------------------------------------------------------------ +@router.get("/library/skills") +async def skills_list(request: Request, db: Db, user: RequiredUser, q: str = "", page: int = 1): + if q.strip(): + rows = skills_service.search(db, user, q, limit=PAGE_SIZE) + pager = {"page": 1, "pages": 1, "total": len(rows)} + else: + rows, pager = _page(db, skills_service.visible(db, user).order_by(Skill.name), page) + return render( + request, + "library/skills.html", + { + "section": "skills", + "skills": rows, + "q": q, + "pager": pager, + **sidebar_context(db, user), + }, + ) + + +@router.get("/library/skills/new") +async def new_skill(request: Request, db: Db, user: RequiredUser): + return render( + request, + "library/skill_detail.html", + {"section": "skills", "skill": None, **sidebar_context(db, user)}, + ) + + +@router.get("/library/skills/{skill_id}") +async def skill_detail(request: Request, db: Db, user: RequiredUser, skill_id: str): + skill = skills_service.get(db, skill_id, user) + if skill is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That skill is not available.") + return render( + request, + "library/skill_detail.html", + { + "section": "skills", + "skill": skill, + "revisions": skill.revisions, + **_shared_context(db, user, skill), + **sidebar_context(db, user), + }, + ) + + +@router.post("/api/library/skills") +async def create_skill( + db: Db, + user: RequiredUser, + name: str = Form(""), + description: str = Form(""), + body: str = Form(""), +) -> Response: + try: + skill = skills_service.create( + db, owner=user, name=name, description=description, body=body, author=AUTHOR_USER + ) + except skills_service.SkillError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc + return RedirectResponse(f"/library/skills/{skill.id}", status_code=status.HTTP_303_SEE_OTHER) + + +@router.post("/api/library/skills/{skill_id}") +async def update_skill(request: Request, db: Db, user: RequiredUser, skill_id: str) -> Response: + skill = skills_service.get(db, skill_id, user) + if skill is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That skill is not available.") + if not sharing.can_write(skill, user): + raise HTTPException(status.HTTP_403_FORBIDDEN, "That skill is not yours to change.") + + form = await request.form() + skills_service.update( + db, + skill, + description=str(form.get("description", "")), + body=str(form.get("body", "")), + enabled="enabled" in form, + author=AUTHOR_USER, + note="edited by hand", + ) + _apply_shares(db, user, skill, form) + return RedirectResponse(f"/library/skills/{skill.id}", status_code=status.HTTP_303_SEE_OTHER) + + +@router.post("/api/library/skills/{skill_id}/revert/{revision_id}") +async def revert_skill( + db: Db, user: RequiredUser, skill_id: str, revision_id: str +) -> Response: + skill = skills_service.get(db, skill_id, user) + if skill is None or not sharing.can_write(skill, user): + raise HTTPException(status.HTTP_404_NOT_FOUND, "That skill is not available.") + revision = db.get(SkillRevision, revision_id) + if revision is None or revision.skill_id != skill.id: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That revision no longer exists.") + + skills_service.revert(db, skill, revision, author=AUTHOR_USER) + return RedirectResponse(f"/library/skills/{skill.id}", status_code=status.HTTP_303_SEE_OTHER) + + +@router.post("/api/library/skills/{skill_id}/delete") +async def delete_skill(db: Db, user: RequiredUser, skill_id: str) -> Response: + skill = skills_service.get(db, skill_id, user) + if skill is None or not sharing.can_write(skill, user): + raise HTTPException(status.HTTP_404_NOT_FOUND, "That skill is not available.") + skills_service.delete(db, skill) + return RedirectResponse("/library/skills", status_code=status.HTTP_303_SEE_OTHER) + + +# --- Memory ------------------------------------------------------------------ +# Lives in Settings rather than in the library: it is a set of short facts about +# the reader, not content they collected. +@router.post("/api/library/memories") +async def add_memory(db: Db, user: RequiredUser, content: str = Form("")) -> Response: + try: + memories_service.add(db, owner=user, content=content, author=AUTHOR_USER) + except ValueError as exc: + from urllib.parse import quote + + return RedirectResponse( + f"/settings?error={quote(str(exc))}", status_code=status.HTTP_303_SEE_OTHER + ) + return RedirectResponse( + "/settings?saved=Memory+added.", status_code=status.HTTP_303_SEE_OTHER + ) + + +@router.post("/api/library/memories/{memory_id}") +async def update_memory( + db: Db, user: RequiredUser, memory_id: str, content: str = Form("") +) -> Response: + memory = memories_service.get(db, memory_id, user) + if memory is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That memory no longer exists.") + try: + memories_service.update(db, memory, content) + except ValueError as exc: + raise HTTPException(status.HTTP_400_BAD_REQUEST, str(exc)) from exc + return RedirectResponse( + "/settings?saved=Memory+updated.", status_code=status.HTTP_303_SEE_OTHER + ) + + +@router.post("/api/library/memories/{memory_id}/delete") +async def delete_memory(db: Db, user: RequiredUser, memory_id: str) -> Response: + memory = memories_service.get(db, memory_id, user) + if memory is None: + raise HTTPException(status.HTTP_404_NOT_FOUND, "That memory no longer exists.") + memories_service.delete(db, memory) + return RedirectResponse( + "/settings?saved=Memory+removed.", status_code=status.HTTP_303_SEE_OTHER + ) diff --git a/src/lembas/api/pages.py b/src/lembas/api/pages.py index 6de6d42..7ec32a0 100644 --- a/src/lembas/api/pages.py +++ b/src/lembas/api/pages.py @@ -46,9 +46,12 @@ def _chat_context(db: DBSession, user: User, chat: Chat | None) -> dict: } -def _sidebar_context(db: DBSession, user: User) -> dict: +def sidebar_context(db: DBSession, user: User) -> dict: """Folder tree plus the chats that belong to no folder. + Public because every page carrying the chat sidebar needs it, which now + includes the library. + Only root folders are queried; children come through the relationship and render recursively in the template. """ @@ -178,7 +181,7 @@ async def chat_index(request: Request, db: Db, user: RequiredUser, model: str = "bodies": {}, **context, "current_model": preselected, - **_sidebar_context(db, user), + **sidebar_context(db, user), }, ) @@ -236,7 +239,7 @@ async def chat_detail(request: Request, db: Db, user: RequiredUser, chat_id: str "inherited_prompt": inherited, "inherited_from": inherited_from, **_chat_context(db, user, chat), - **_sidebar_context(db, user), + **sidebar_context(db, user), }, ) @@ -250,6 +253,7 @@ async def settings_page( saved: str = "", ): from lembas.api.audio import available_voices + from lembas.services.library import memories as memories_service context = _chat_context(db, user, None) # Fetched here rather than by the template so a speech server that is down @@ -267,7 +271,9 @@ async def settings_page( "saved": saved, "voices": voices, "voice_error": voice_error, + "memories": memories_service.all_for(db, user), + "memory_limit": memories_service.MAX_MEMORY_CHARS, **context, - **_sidebar_context(db, user), + **sidebar_context(db, user), }, ) diff --git a/src/lembas/db/migrations.py b/src/lembas/db/migrations.py index eac9d4f..83d99a6 100644 --- a/src/lembas/db/migrations.py +++ b/src/lembas/db/migrations.py @@ -96,6 +96,93 @@ def _add_column_sql(table: Table, column: Column, dialect) -> str | None: return " ".join(parts) +# --- Full-text search -------------------------------------------------------- +# The library stores are searched rather than listed, and LIKE over a few +# hundred documents ranks nothing and matches badly. SQLite ships FTS5, so the +# index costs no dependency and works offline like everything else here. +# +# These are the one part of the schema this module's model-diffing cannot +# derive: an FTS5 virtual table is not a SQLAlchemy model, has no columns to +# compare, and needs triggers to stay in step with the table it shadows. So it +# is written out -- but written out *idempotently*, with IF NOT EXISTS +# throughout, which keeps it the same kind of thing as the column sync: run it +# at every startup and it converges. +# +# `content=` makes each index external-content: the text is not stored twice, +# and the triggers below are what the FTS5 documentation calls for to keep an +# external-content index correct through updates and deletes. +FTS_INDEXES: tuple[tuple[str, str, tuple[str, ...]], ...] = ( + ("documents_fts", "documents", ("title", "description", "extracted_text")), + ("notes_fts", "notes", ("title", "body")), + ("skills_fts", "skills", ("name", "description", "body")), +) + + +def _fts_statements(index: str, table: str, columns: tuple[str, ...]) -> list[str]: + # `id` rides along UNINDEXED so a match can be turned straight back into an + # ORM row. The alternative is joining on rowid, which SQLAlchemy models do + # not expose and which changes under VACUUM. + columns = ("id", *columns) + column_list = ", ".join(columns) + declared = ", ".join( + f"{name} UNINDEXED" if name == "id" else name for name in columns + ) + new_values = ", ".join(f"new.{name}" for name in columns) + old_values = ", ".join(f"old.{name}" for name in columns) + + return [ + f"CREATE VIRTUAL TABLE IF NOT EXISTS {index} USING fts5(" + f"{declared}, content='{table}', content_rowid='rowid')", + # 'delete' rows carry the old values because an external-content index + # cannot look them up itself once the source row has gone. + f"""CREATE TRIGGER IF NOT EXISTS {index}_ai AFTER INSERT ON {table} BEGIN + INSERT INTO {index}(rowid, {column_list}) VALUES (new.rowid, {new_values}); + END""", + f"""CREATE TRIGGER IF NOT EXISTS {index}_ad AFTER DELETE ON {table} BEGIN + INSERT INTO {index}({index}, rowid, {column_list}) + VALUES ('delete', old.rowid, {old_values}); + END""", + f"""CREATE TRIGGER IF NOT EXISTS {index}_au AFTER UPDATE ON {table} BEGIN + INSERT INTO {index}({index}, rowid, {column_list}) + VALUES ('delete', old.rowid, {old_values}); + INSERT INTO {index}(rowid, {column_list}) VALUES (new.rowid, {new_values}); + END""", + ] + + +def ensure_fts(engine: Engine) -> list[str]: + """Create the search indexes and their triggers if they are missing. + + Returns the indexes it created. A failure here is logged and swallowed: + search degrading to "finds nothing" is bad, but it is much better than the + application refusing to start. + """ + created: list[str] = [] + inspector = inspect(engine) + known = set(inspector.get_table_names()) + + with engine.begin() as connection: + for index, table, columns in FTS_INDEXES: + if table not in known: + continue + fresh = index not in known + for statement in _fts_statements(index, table, columns): + connection.execute(text(statement)) + if fresh: + # Backfill anything already in the table. Only on creation -- + # the triggers keep it current from then on. + column_list = ", ".join(("id", *columns)) + connection.execute( + text( + f"INSERT INTO {index}(rowid, {column_list}) " + f"SELECT rowid, {column_list} FROM {table}" + ) + ) + created.append(index) + + return created + + def sync_schema(engine: Engine) -> list[str]: """Bring the database up to the declared schema. Returns what it changed.""" import lembas.db.models # noqa: F401 (registers every table on the metadata) @@ -125,6 +212,12 @@ def sync_schema(engine: Engine) -> list[str]: changes.append(f"add column {table.name}.{column.name}") log.info("schema: %s", statement) + try: + for index in ensure_fts(engine): + changes.append(f"create search index {index}") + except Exception: # noqa: BLE001 - search is not worth refusing to start over + log.exception("could not create the full-text search indexes") + if changes: log.info("schema synchronised: %d change(s)", len(changes)) for step in MANUAL_STEPS: diff --git a/src/lembas/db/models/__init__.py b/src/lembas/db/models/__init__.py index 4a8e506..c6edaeb 100644 --- a/src/lembas/db/models/__init__.py +++ b/src/lembas/db/models/__init__.py @@ -21,6 +21,23 @@ from lembas.db.models.chat import ( Message, ) from lembas.db.models.connection import Connection, Model, model_groups +from lembas.db.models.library import ( + AUTHOR_MODEL, + AUTHOR_USER, + PRINCIPAL_GROUP, + PRINCIPAL_USER, + RESOURCE_DOCUMENT, + RESOURCE_NOTE, + RESOURCE_SKILL, + SOURCE_LINK, + SOURCE_UPLOAD, + Document, + Memory, + Note, + Share, + Skill, + SkillRevision, +) from lembas.db.models.setting import Setting from lembas.db.models.user import ( ROLE_ADMIN, @@ -32,25 +49,40 @@ from lembas.db.models.user import ( ) __all__ = [ + "AUTHOR_MODEL", + "AUTHOR_USER", "Attachment", "KIND_DOCUMENT", "KIND_IMAGE", "KIND_TEXT", + "PRINCIPAL_GROUP", + "PRINCIPAL_USER", + "RESOURCE_DOCUMENT", + "RESOURCE_NOTE", + "RESOURCE_SKILL", "ROLE_ADMIN", "ROLE_ASSISTANT", "ROLE_PENDING", "ROLE_SYSTEM", "ROLE_TOOL", "ROLE_USER", + "SOURCE_LINK", + "SOURCE_UPLOAD", "Chat", "Connection", + "Document", "Folder", "Group", + "Memory", "Message", "Model", - "model_groups", + "Note", "Session", "Setting", + "Share", + "Skill", + "SkillRevision", "User", + "model_groups", "user_groups", ] diff --git a/src/lembas/db/models/library.py b/src/lembas/db/models/library.py new file mode 100644 index 0000000..d97d35e --- /dev/null +++ b/src/lembas/db/models/library.py @@ -0,0 +1,228 @@ +"""What the model can reach for: knowledge, notes, memory and skills. + +Four stores rather than one, because they differ in the two ways that matter -- +who writes a record, and how a record reaches the model: + +* **Document** is uploaded by a person and searched by the model. It is the + only one holding a file, and it is deliberately shaped like ``Attachment``: + both come out of ``services.files.prepare`` and carry the same processed + content. +* **Note** is written by the model and edited by a person. Long enough that it + has to be searched rather than injected. +* **Memory** is one short fact, and *is* injected -- every one of them, every + turn, up to a budget. Anything that would not survive that treatment belongs + in a note. +* **Skill** is a named instruction document. Its description is injected so the + model knows the skill exists; the body is fetched only when it decides to use + it, which is what keeps a hundred skills affordable. + +Everything except Memory can be shared -- see ``Share`` below and +``services.sharing``. Memory cannot: a record about a person is not content to +hand round, and "share my memories with the team" is a question nobody asked. +""" + +from __future__ import annotations + +from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column, relationship + +from lembas.db.base import Base, Timestamps, UUIDPrimaryKey + +# Who wrote a record. Not decoration: a skill the model wrote itself is the one +# worth looking at twice when its behaviour changes unexpectedly. +AUTHOR_USER = "user" +AUTHOR_MODEL = "model" + +# Where a document came from. +SOURCE_UPLOAD = "upload" +SOURCE_LINK = "link" + +# Resource kinds that can be shared. Values are stored, so they are part of the +# schema rather than an implementation detail. +RESOURCE_DOCUMENT = "document" +RESOURCE_NOTE = "note" +RESOURCE_SKILL = "skill" + +PRINCIPAL_USER = "user" +PRINCIPAL_GROUP = "group" + + +class Document(UUIDPrimaryKey, Timestamps, Base): + """One item in a knowledge library: a file, an image or a saved web page. + + The content columns mirror ``Attachment`` exactly because both are produced + by ``services.files.prepare`` -- images downscaled, PDF text extracted once, + type decided by sniffing bytes. Keeping the shapes identical is what lets a + document be attached to a message by copying rather than converting. + """ + + __tablename__ = "documents" + + owner_id: Mapped[str] = mapped_column( + String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + + title: Mapped[str] = mapped_column(String(300), nullable=False) + description: Mapped[str] = mapped_column(Text, default="") + + source: Mapped[str] = mapped_column(String(16), default=SOURCE_UPLOAD, nullable=False) + # Set for a saved web page, so it can be re-fetched and cited. + source_url: Mapped[str] = mapped_column(Text, default="") + + # --- The same content columns as Attachment --- + filename: Mapped[str] = mapped_column(String(300), default="") + stored_name: Mapped[str] = mapped_column(String(120), default="") + media_type: Mapped[str] = mapped_column(String(100), default="") + size_bytes: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + kind: Mapped[str] = mapped_column(String(16), default="text", nullable=False) + width: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + height: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + extracted_text: Mapped[str] = mapped_column(Text, default="") + pages: Mapped[int] = mapped_column(Integer, default=0, nullable=False) + truncated: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + extraction_error: Mapped[str] = mapped_column(Text, default="") + + @property + def is_image(self) -> bool: + return self.kind == "image" + + @property + def human_size(self) -> str: + size = float(self.size_bytes) + for unit in ("B", "KB", "MB"): + if size < 1024 or unit == "MB": + return f"{size:.0f} {unit}" if unit == "B" else f"{size:.1f} {unit}" + size /= 1024 + return f"{size:.1f} MB" + + def __repr__(self) -> str: + return f"" + + +class Note(UUIDPrimaryKey, Timestamps, Base): + """Something the model wrote down, or a person did. + + Longer and more specific than a memory. Not injected: a handful of notes + would fill a context window on their own, so the model searches for the one + it needs. + """ + + __tablename__ = "notes" + + owner_id: Mapped[str] = mapped_column( + String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + title: Mapped[str] = mapped_column(String(300), nullable=False) + body: Mapped[str] = mapped_column(Text, default="") + author: Mapped[str] = mapped_column(String(16), default=AUTHOR_USER, nullable=False) + + def __repr__(self) -> str: + return f"" + + +class Memory(UUIDPrimaryKey, Timestamps, Base): + """One short fact, in front of the model on every turn. + + Deliberately not shareable and deliberately small. The length cap is + enforced in the service rather than by the column, so an over-long write + from a tool is trimmed with an explanation instead of failing the turn. + """ + + __tablename__ = "memories" + + owner_id: Mapped[str] = mapped_column( + String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + content: Mapped[str] = mapped_column(Text, nullable=False) + author: Mapped[str] = mapped_column(String(16), default=AUTHOR_MODEL, nullable=False) + + def __repr__(self) -> str: + return f"" + + +class Skill(UUIDPrimaryKey, Timestamps, Base): + """A named set of instructions the model can choose to follow. + + `description` is the load-bearing field: it is what gets injected, and it is + the only thing the model has to decide whether the skill is relevant. The + body is fetched with a tool. + """ + + __tablename__ = "skills" + __table_args__ = (UniqueConstraint("owner_id", "name"),) + + owner_id: Mapped[str] = mapped_column( + String(32), ForeignKey("users.id", ondelete="CASCADE"), nullable=False, index=True + ) + # Slug, referenced by the model when it asks for the body. + name: Mapped[str] = mapped_column(String(120), nullable=False) + description: Mapped[str] = mapped_column(Text, default="") + body: Mapped[str] = mapped_column(Text, default="") + enabled: Mapped[bool] = mapped_column(Boolean, default=True, nullable=False) + author: Mapped[str] = mapped_column(String(16), default=AUTHOR_USER, nullable=False) + + revisions: Mapped[list[SkillRevision]] = relationship( + back_populates="skill", + cascade="all, delete-orphan", + order_by="SkillRevision.created_at.desc()", + ) + + def __repr__(self) -> str: + return f"" + + +class SkillRevision(UUIDPrimaryKey, Timestamps, Base): + """The state of a skill before a change. + + A model may rewrite its own skills, so every write snapshots what was there + first. That is the whole safety story for self-modification: not a gate, but + a record and a way back. + """ + + __tablename__ = "skill_revisions" + + skill_id: Mapped[str] = mapped_column( + String(32), ForeignKey("skills.id", ondelete="CASCADE"), nullable=False, index=True + ) + description: Mapped[str] = mapped_column(Text, default="") + body: Mapped[str] = mapped_column(Text, default="") + # Who made the change this revision is the "before" of. + author: Mapped[str] = mapped_column(String(16), default=AUTHOR_USER, nullable=False) + note: Mapped[str] = mapped_column(String(200), default="") + + skill: Mapped[Skill] = relationship(back_populates="revisions") + + +class Share(UUIDPrimaryKey, Timestamps, Base): + """One grant of access to one resource. + + A single table across documents, notes and skills rather than three + association tables, because the rule is identical in all three cases and + ``services.sharing`` is the only thing that reads it. + + A grant, never a denial -- the same principle as group permissions. Somebody + who cannot see a resource simply has no row here. + """ + + __tablename__ = "shares" + __table_args__ = ( + UniqueConstraint( + "resource_type", "resource_id", "principal_type", "principal_id" + ), + ) + + resource_type: Mapped[str] = mapped_column(String(16), nullable=False) + resource_id: Mapped[str] = mapped_column(String(32), nullable=False) + + principal_type: Mapped[str] = mapped_column(String(16), nullable=False) + # No foreign key: this column points at users or groups depending on + # principal_type, and SQLite cannot express that. services.sharing deletes + # dangling rows when a user or group goes. + principal_id: Mapped[str] = mapped_column(String(32), nullable=False) + + def __repr__(self) -> str: + return f" {self.principal_type}>" + + +Index("ix_shares_resource", Share.resource_type, Share.resource_id) +Index("ix_shares_principal", Share.principal_type, Share.principal_id) diff --git a/src/lembas/main.py b/src/lembas/main.py index 8eb254e..ea665b9 100644 --- a/src/lembas/main.py +++ b/src/lembas/main.py @@ -23,6 +23,7 @@ from lembas.api import ( chats, files, folders, + library, pages, preferences, ) @@ -98,6 +99,7 @@ def create_app() -> FastAPI: app.include_router(audio.router) app.include_router(files.router) app.include_router(folders.router) + app.include_router(library.router) app.include_router(admin.router) app.include_router(admin_users.router) app.include_router(admin_models.router) diff --git a/src/lembas/security/permissions.py b/src/lembas/security/permissions.py index 3a5c745..0b6dc90 100644 --- a/src/lembas/security/permissions.py +++ b/src/lembas/security/permissions.py @@ -100,6 +100,51 @@ PERMISSION_DEFS: tuple[PermissionDef, ...] = ( True, "Audio", ), + PermissionDef( + "library.use", + "Use the library", + "Keep knowledge documents, notes, memories and skills of their own.", + True, + "Library", + ), + PermissionDef( + "library.share", + "Share library items", + "Give other people, or a group, access to their documents, notes and " + "skills. Sharing grants reading only.", + False, + "Library", + ), + PermissionDef( + "tools.knowledge", + "Search their knowledge", + "Let a model search the documents this user has collected.", + True, + "Library", + ), + PermissionDef( + "tools.notes", + "Read and write notes", + "Let a model keep its own notes for this user, and read them back later.", + True, + "Library", + ), + PermissionDef( + "tools.memory", + "Remember things", + "Let a model record short facts about this user, shown to it on every " + "turn.", + True, + "Library", + ), + PermissionDef( + "tools.skills", + "Use and write skills", + "Let a model follow saved instructions, and write new ones. Every " + "change is recorded and can be rolled back.", + True, + "Library", + ), ) PERMISSION_KEYS = tuple(d.key for d in PERMISSION_DEFS) diff --git a/src/lembas/services/chat.py b/src/lembas/services/chat.py index ece70f9..8a5b80c 100644 --- a/src/lembas/services/chat.py +++ b/src/lembas/services/chat.py @@ -156,15 +156,22 @@ def effective_system_prompt(db: DBSession, chat: Chat) -> str: def build_messages( - db: DBSession, chat: Chat, *, upto: Message | None = None, vision: bool = False + db: DBSession, + chat: Chat, + *, + upto: Message | None = None, + vision: bool = False, + system_prompt: str | None = None, ) -> list[dict]: """Assemble the message list to send upstream. `upto` excludes the placeholder assistant row being generated into, and - everything after it. + everything after it. `system_prompt` overrides what would otherwise be + resolved, which is how the harness gets in front of the authored prompt + without this function knowing anything about tools. """ payload: list[dict[str, Any]] = [] - system = effective_system_prompt(db, chat) + system = effective_system_prompt(db, chat) if system_prompt is None else system_prompt if system: payload.append({"role": ROLE_SYSTEM, "content": system}) @@ -186,15 +193,40 @@ def build_messages( return payload -def model_supports(db: DBSession, chat: Chat, capability: str) -> bool: - """Whether the chat's current model is marked as having a capability.""" - model = db.scalar( +def model_for(db: DBSession, chat: Chat) -> Model | None: + """The Model row a chat is using, or None if it has gone. + + Looked up by id rather than held as a foreign key, for the same reason + resolve_endpoint does: chats store the model as text so history survives an + administrator deleting a connection. + """ + return db.scalar( select(Model).where(Model.model_id == chat.model_id).order_by(Model.position) ) + + +def model_supports(db: DBSession, chat: Chat, capability: str) -> bool: + """Whether the chat's current model is marked as having a capability.""" + model = model_for(db, chat) return bool(model and (model.capabilities_json or {}).get(capability)) -def build_request(db: DBSession, chat: Chat, *, upto: Message | None = None) -> dict[str, Any]: +def build_request( + db: DBSession, + chat: Chat, + *, + upto: Message | None = None, + tools: list[dict[str, Any]] | None = None, + user=None, +) -> dict[str, Any]: + """The whole request body, tools and harness included. + + Composed here rather than in the generation loop so that "what gets sent" + has one answer, and so the harness cannot be forgotten by a future caller + that offers tools. + """ + from lembas.services import harness as harness_service + params = { key: value for key, value in (chat.params_json or {}).items() @@ -204,11 +236,29 @@ def build_request(db: DBSession, chat: Chat, *, upto: Message | None = None) -> # vision. Sending them to one that has not is not a graceful degradation: # most endpoints reject the whole request. vision = model_supports(db, chat, "vision") - return { + + if user is None: + from lembas.db.models import User + + user = db.get(User, chat.user_id) + + # The harness describes the tools; the authored prompt describes the + # behaviour. See services/harness.py for why these are joined rather than + # being two competing layers. + system = harness_service.join( + harness_service.compose(db, user, tools), effective_system_prompt(db, chat) + ) + + body: dict[str, Any] = { "model": chat.model_id, - "messages": build_messages(db, chat, upto=upto, vision=vision), + "messages": build_messages( + db, chat, upto=upto, vision=vision, system_prompt=system + ), **params, } + if tools: + body["tools"] = tools + return body def default_model(db: DBSession, user=None) -> tuple[str, str] | None: diff --git a/src/lembas/services/fetch.py b/src/lembas/services/fetch.py new file mode 100644 index 0000000..6fa064d --- /dev/null +++ b/src/lembas/services/fetch.py @@ -0,0 +1,221 @@ +"""Fetching a web page so it can be kept, or read to a model. + +Two things this deliberately does not do. + +**It does not try to be clever about extraction.** No readability heuristics, no +main-column detection: script and style go, tags are dropped, whitespace is +collapsed. A clever extractor that silently discards the part somebody wanted is +worse than a plain one that keeps everything, and it would be a dependency. + +**It does not trust the URL.** This runs on a server that can very likely reach +a router's admin page, a metadata endpoint, and every other service on the same +machine -- LLeMbas itself included. A fetcher that takes a URL from a user, or +worse from a model, is a request-forgery hole unless something stops it, so +addresses are checked after resolution and redirects are followed by hand. +""" + +from __future__ import annotations + +import ipaddress +import logging +import re +import socket +from dataclasses import dataclass +from urllib.parse import urlparse, urlunparse + +import httpx +import nh3 + +log = logging.getLogger(__name__) + +# Pages are kept as text, so the ceiling is about what is worth reading rather +# than what will fit on disk. +MAX_PAGE_BYTES = 5 * 1024 * 1024 +MAX_TEXT_CHARS = 120_000 +MAX_REDIRECTS = 5 +TIMEOUT = 20.0 + +# Sent because a plain httpx user agent is blocked by a good number of sites, +# and being honest about what this is beats impersonating a browser. +USER_AGENT = "Mozilla/5.0 (compatible; LLeMbas/1.0; +https://github.com/homer/LLeMbas)" + +# goes wholesale, which takes script, style and the title with it. The +# title is pulled out of the raw HTML first, so removing it here is what stops +# it appearing again as the opening line of the body. +_DROPPED = re.compile( + r"<(head|script|style|noscript|template|svg)\b[^>]*>.*?", + re.IGNORECASE | re.DOTALL, +) +_TITLE = re.compile(r"]*>(.*?)", re.IGNORECASE | re.DOTALL) +# Tags that end a line of prose. Turning them into newlines before the tags are +# stripped is the difference between readable text and one enormous paragraph. +_BREAKS = re.compile( + r"|", + re.IGNORECASE, +) + + +class FetchError(Exception): + """A refused or failed fetch, with a message fit to show a user.""" + + def __init__(self, message: str) -> None: + super().__init__(message) + self.message = message + + +@dataclass +class Fetched: + url: str + title: str + text: str + truncated: bool = False + + +def _is_public(address: str) -> bool: + """Whether an IP is one this server should be willing to fetch from. + + Loopback reaches LLeMbas and every other local service. Private ranges reach + the rest of the network the server sits on. Link-local covers cloud metadata + endpoints, which is where credentials live. + """ + try: + ip = ipaddress.ip_address(address) + except ValueError: + return False + return not ( + ip.is_private + or ip.is_loopback + or ip.is_link_local + or ip.is_multicast + or ip.is_reserved + or ip.is_unspecified + ) + + +def check_url(url: str, *, allow_private: bool = False) -> str: + """Validate a URL and return it normalised. Raises FetchError if refused.""" + try: + parsed = urlparse(url.strip()) + except ValueError as exc: + raise FetchError("That does not look like a URL.") from exc + + if parsed.scheme not in ("http", "https"): + raise FetchError("Only http and https addresses can be fetched.") + if not parsed.hostname: + raise FetchError("That URL has no host.") + + if not allow_private: + try: + # Resolved, not parsed: a hostname pointing at 127.0.0.1 is the + # obvious way past a check that only looks at the text of the URL. + resolved = socket.getaddrinfo(parsed.hostname, None) + except socket.gaierror as exc: + raise FetchError(f"Could not resolve {parsed.hostname}.") from exc + + addresses = {info[4][0] for info in resolved} + # Every address, not any: a name resolving to one public and one private + # address must not be usable to reach the private one. + if not addresses or not all(_is_public(address) for address in addresses): + raise FetchError( + f"{parsed.hostname} resolves to a private or local address. " + "An administrator can allow this under Admin → Web search if " + "fetching from this network is intended." + ) + + return urlunparse(parsed) + + +def html_to_text(html: str) -> tuple[str, str]: + """Reduce a page to (title, text).""" + title_match = _TITLE.search(html) + title = "" + if title_match: + title = " ".join(nh3.clean(title_match.group(1), tags=set()).split()) + + body = _DROPPED.sub(" ", html) + body = _BREAKS.sub("\n", body) + # nh3 with no allowed tags leaves the text and escapes nothing structural; + # it is the same sanitiser the rest of the application trusts. + body = nh3.clean(body, tags=set(), attributes={}) + + import html as html_module + + body = html_module.unescape(body) + lines = [" ".join(line.split()) for line in body.splitlines()] + # Collapse runs of blank lines, which a stripped page is mostly made of. + text, blank = [], False + for line in lines: + if line: + text.append(line) + blank = False + elif not blank: + text.append("") + blank = True + + return title, "\n".join(text).strip() + + +async def fetch(url: str, *, allow_private: bool = False) -> Fetched: + """Retrieve a page and reduce it to text. + + Redirects are followed by hand so every hop can be checked. httpx's own + following would validate the first address and then happily land on + localhost. + """ + current = check_url(url, allow_private=allow_private) + + try: + async with httpx.AsyncClient( + timeout=TIMEOUT, follow_redirects=False, headers={"User-Agent": USER_AGENT} + ) as client: + for _ in range(MAX_REDIRECTS + 1): + response = await client.get(current) + + if response.is_redirect: + location = response.headers.get("location", "") + if not location: + raise FetchError("That page redirected to nowhere.") + current = check_url( + str(response.url.join(location)), allow_private=allow_private + ) + continue + + if response.status_code >= 400: + raise FetchError( + f"{current} returned HTTP {response.status_code}." + ) + break + else: + raise FetchError("That page redirected too many times.") + except httpx.RequestError as exc: + raise FetchError(f"Could not reach {current}: {exc}") from exc + + payload = response.content[:MAX_PAGE_BYTES] + content_type = response.headers.get("content-type", "") + + if "html" in content_type or payload[:512].lstrip()[:1] == b"<": + title, text = html_to_text(payload.decode(response.encoding or "utf-8", "replace")) + elif content_type.startswith("text/") or not content_type: + title, text = "", payload.decode(response.encoding or "utf-8", "replace") + else: + raise FetchError( + f"That address is {content_type or 'not text'}, which cannot be saved " + "as a page. Attach it as a file instead." + ) + + truncated = len(text) > MAX_TEXT_CHARS + if not text.strip(): + raise FetchError( + "Nothing readable was found at that address. It may be a page that " + "builds itself with JavaScript, which this cannot run." + ) + + return Fetched( + url=current, + title=title or urlparse(current).netloc or current, + text=text[:MAX_TEXT_CHARS], + truncated=truncated, + ) + + +__all__ = ["FetchError", "Fetched", "check_url", "fetch", "html_to_text"] diff --git a/src/lembas/services/files.py b/src/lembas/services/files.py index 06eb6d8..8e2787e 100644 --- a/src/lembas/services/files.py +++ b/src/lembas/services/files.py @@ -325,6 +325,84 @@ def store( return attachment +def store_text( + db: DBSession, + *, + user_id: str, + chat_id: str | None, + filename: str, + text: str, + truncated: bool = False, + source_note: str = "", +) -> Attachment: + """Attach text that did not arrive as a file -- a fetched web page. + + Written to disk like any other attachment so it can be downloaded and so + there is one cleanup path, rather than a second kind of attachment that + exists only in the database. + """ + body = text[:MAX_EXTRACTED_CHARS] + payload = body.encode("utf-8") + + stored_name = f"{secrets.token_hex(16)}.txt" + (attachments_dir() / stored_name).write_bytes(payload) + + attachment = Attachment( + user_id=user_id, + chat_id=chat_id, + filename=safe_display_name(filename), + stored_name=stored_name, + media_type="text/plain", + size_bytes=len(payload), + kind=KIND_TEXT, + # The URL leads the text so the model can cite it, and so the reader + # can see where an attachment called "Some Page.txt" came from. + extracted_text=f"Source: {source_note}\n\n{body}" if source_note else body, + truncated=truncated, + ) + db.add(attachment) + db.commit() + return attachment + + +def copy_document( + db: DBSession, *, user_id: str, chat_id: str | None, document +) -> Attachment: + """Copy a library document into a message being composed. + + A copy rather than a reference. History must not change under a conversation + because a document was edited or deleted afterwards -- the same reason text + is extracted once at upload instead of per request. The bytes are duplicated + too, so deleting the document cannot leave a message pointing at nothing. + """ + from lembas.services.library import documents as documents_service + + stored_name = "" + source = documents_service.stored_path(document.stored_name) + if source is not None: + stored_name = f"{secrets.token_hex(16)}{Path(source.name).suffix}" + (attachments_dir() / stored_name).write_bytes(source.read_bytes()) + + attachment = Attachment( + user_id=user_id, + chat_id=chat_id, + filename=document.filename or f"{document.title}.txt", + stored_name=stored_name, + media_type=document.media_type, + size_bytes=document.size_bytes, + kind=document.kind, + width=document.width, + height=document.height, + extracted_text=document.extracted_text, + pages=document.pages, + truncated=document.truncated, + extraction_error=document.extraction_error, + ) + db.add(attachment) + db.commit() + return attachment + + def delete(db: DBSession, attachment: Attachment) -> None: path = stored_path(attachment.stored_name) if path is not None: diff --git a/src/lembas/services/generation.py b/src/lembas/services/generation.py index f55bda4..7883d2a 100644 --- a/src/lembas/services/generation.py +++ b/src/lembas/services/generation.py @@ -25,7 +25,6 @@ from datetime import UTC, datetime, timedelta from lembas.db.models import ROLE_ASSISTANT, ROLE_USER, Chat, Message, User from lembas.db.session import session_scope from lembas.services import chat as chat_service -from lembas.services import settings_store from lembas.services import tools as tools_service from lembas.services.llm.openai_client import ( LLMError, @@ -167,16 +166,16 @@ async def _run(generation: Generation) -> None: return endpoint, model_id = chat_service.resolve_endpoint(db, chat) - payload = chat_service.build_request(db, chat, upto=message) - question = _question_from(payload) - needs_title = not chat.title_generated + owner = db.get(User, chat.user_id) # Read while the session is open: everything below outlives it. - offered = tools_service.enabled_tools(db, chat, db.get(User, chat.user_id)) - search_config = settings_store.search(db) if offered else {} - - if offered: - payload = {**payload, "tools": offered} + offered = tools_service.enabled_tools(db, chat, owner) + payload = chat_service.build_request( + db, chat, upto=message, tools=offered, user=owner + ) + question = _question_from(payload) + needs_title = not chat.title_generated + tool_context = tools_service.context_for(db, owner) for round_number in range(tools_service.MAX_ROUNDS + 1): accumulator = tools_service.ToolCallAccumulator() @@ -247,7 +246,7 @@ async def _run(generation: Generation) -> None: ] for call in calls: outcome = await tools_service.run_tool( - search_config, call["name"], call["arguments"] + tool_context, call["name"], call["arguments"] ) generation.tool_events.append(outcome.event) generation.touch() diff --git a/src/lembas/services/harness.py b/src/lembas/services/harness.py new file mode 100644 index 0000000..ef7059e --- /dev/null +++ b/src/lembas/services/harness.py @@ -0,0 +1,161 @@ +"""Telling the model how to use what it has been given. + +A model handed a `tools` array will often ignore it. It answers from recall +because that is what it was trained to do, and nothing in the request suggests +otherwise. The harness is the part of the prompt that says otherwise: one line +per tool about *when* to reach for it, the memories, and the list of skills +available. + +**On the "system prompts are precedence, not concatenation" rule.** That rule +governs the three authored layers -- instance, model, chat -- and it is +untouched here: exactly one of them still wins, and +``chat.effective_system_prompt`` still decides which. This is a different axis. +It describes the machinery rather than the behaviour, nobody authored it, and +there is nothing for it to disagree with. So it is prepended to whichever +authored prompt won, inside one system message, under a heading that makes the +seam obvious. + +One system message rather than two because several endpoints reject a second +one. The authored prompt goes last, where it is closest to the conversation. + +Nothing is emitted for a model with no tools and no memories: an empty harness +is worse than none, being tokens that say only that there is nothing to say. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy.orm import Session as DBSession + +from lembas.db.models import User +from lembas.services.library import memories as memories_service +from lembas.services.library import skills as skills_service + +log = logging.getLogger(__name__) + +# Keyed by tool family, so a family that is off contributes nothing. Written as +# guidance rather than rules: a model told "you MUST search" searches for the +# capital of France. +GUIDANCE: dict[str, str] = { + "web_search": ( + "- Look things up rather than trusting your recall, whenever the answer " + "depends on current facts, on details you are not certain of, or on " + "anything that may have changed. If the first results are thin or " + "beside the point, search again with different words instead of " + "answering from them — two or three searches are normal. Say where an " + "answer came from." + ), + "knowledge": ( + "- The user has a library of their own documents. When a question is " + "about their material — their files, their notes on paper, a page they " + "saved — search that before searching the web." + ), + "notes": ( + "- You keep notes across conversations. Search them when a task sounds " + "like one you have done before. Write one when you work something out " + "that would be tedious to work out again: a procedure, a decision and " + "its reasons, a summary of a long document." + ), + # Two variants: the first refers to a heading that only exists when there + # is something under it, and telling a model to consult an absent section + # is a good way to make it invent one. + "memory": ( + "- You can remember durable facts about this person — a preference, a " + "constraint, a name — but not the details of one task, and never " + "anything secret." + ), + "memory_with_records": ( + "- What is listed under “What you know about this person” below was " + "remembered earlier and still applies. Add to it only for durable facts " + "— a preference, a constraint, a name — never for the details of one " + "task, and never for anything secret." + ), + "skills": ( + "- Skills are procedures you have saved. The list below gives only each " + "one's name and when to use it; read the full instructions with " + "skill_get before following one. If you work out a repeatable way to do " + "something, save it as a new skill." + ), +} + +HEADING = "## How to work" + +# A ceiling on the whole block, so that a large library cannot quietly eat the +# context window. Memory and skills have their own caps below this one. +MAX_HARNESS_CHARS = 8000 + + +def _families(tools: list[dict[str, Any]]) -> list[str]: + """Which families are represented in an offered tool list, in a fixed order.""" + from lembas.services.tools import FAMILIES, REGISTRY + + offered = { + REGISTRY[name].family + for tool in tools + if (name := (tool.get("function") or {}).get("name")) in REGISTRY + } + return [family for family in FAMILIES if family in offered] + + +def compose( + db: DBSession, + user: User | None, + tools: list[dict[str, Any]] | None, +) -> str: + """The operational preamble for this request, or "" when there is nothing to say.""" + families = _families(tools or []) + if not families: + return "" + + parts: list[str] = [ + HEADING, + "", + "You have tools. Use them rather than guessing; a wrong answer given " + "confidently is worse than a slower one that was checked.", + "", + ] + # Read before the guidance is assembled, because whether there are any + # memories decides which wording the memory line gets. + block = memories_service.block(db, user) if "memory" in families else "" + + for family in families: + if family == "memory" and block: + parts.append(GUIDANCE["memory_with_records"]) + elif family in GUIDANCE: + parts.append(GUIDANCE[family]) + + if block: + parts += ["", "### What you know about this person", "", block] + + if "skills" in families: + index = skills_service.index_block(db, user) + if index: + parts += [ + "", + "### Skills available", + "", + index, + "", + "Read one with skill_get before following it.", + ] + + text = "\n".join(parts).strip() + if len(text) > MAX_HARNESS_CHARS: + text = text[:MAX_HARNESS_CHARS].rstrip() + "\n…" + return text + + +def join(harness: str, authored: str) -> str: + """Put the harness in front of whichever authored prompt won. + + Separated from `compose` so the precedence between instance, model and chat + stays testable on its own -- this function is the only place the two axes + meet. + """ + if not harness: + return authored + if not authored: + return harness + return f"{harness}\n\n---\n\n{authored}" diff --git a/src/lembas/services/library/__init__.py b/src/lembas/services/library/__init__.py new file mode 100644 index 0000000..48d4c69 --- /dev/null +++ b/src/lembas/services/library/__init__.py @@ -0,0 +1,18 @@ +"""The four stores the model can reach for. + +Knowledge, notes and skills are searched; memory is small enough to be handed +over whole. Everything here answers to one visibility rule -- see +``services.sharing`` -- and nothing here queries a table without it. +""" + +from lembas.services.library.fts import SearchHit, fts_query, search_ids +from lembas.services.library.memories import MAX_MEMORY_CHARS +from lembas.services.library.skills import SKILL_NAME_PATTERN + +__all__ = [ + "MAX_MEMORY_CHARS", + "SKILL_NAME_PATTERN", + "SearchHit", + "fts_query", + "search_ids", +] diff --git a/src/lembas/services/library/documents.py b/src/lembas/services/library/documents.py new file mode 100644 index 0000000..f4f0792 --- /dev/null +++ b/src/lembas/services/library/documents.py @@ -0,0 +1,166 @@ +"""The knowledge library: documents a person has collected. + +Ingestion is deliberately **not** written here. A knowledge document and a chat +attachment are the same processing problem -- sniff the bytes, downscale the +image, extract the PDF once -- so both go through +``services.files.prepare``. Keeping one pipeline is what guarantees the same +PDF produces the same text whichever way it arrived, and it is why `Document` +carries the same content columns as `Attachment`. +""" + +from __future__ import annotations + +import logging +import secrets +from pathlib import Path + +from sqlalchemy import select +from sqlalchemy.orm import Session as DBSession + +from lembas.config import settings +from lembas.db.models import SOURCE_LINK, SOURCE_UPLOAD, Document, User +from lembas.services import files as files_service +from lembas.services import sharing +from lembas.services.fetch import Fetched +from lembas.services.library.fts import search_ids + +log = logging.getLogger(__name__) + +INDEX = "documents_fts" + +# How much of a document's text a search result carries back to the model. A +# whole 100-page extract would swallow the context window; this is enough to +# judge relevance and to answer from, and `knowledge_get` fetches the rest. +SNIPPET_CHARS = 1200 + + +def library_dir() -> Path: + """Where library files live, beside but separate from chat attachments.""" + path = settings.uploads_dir / "library" + path.mkdir(parents=True, exist_ok=True) + return path + + +def stored_path(stored_name: str) -> Path | None: + """Resolve a stored name, refusing anything outside the library directory. + + The same check as ``services.files.stored_path``, against a different root. + """ + if not stored_name or "/" in stored_name or "\\" in stored_name or stored_name.startswith("."): + return None + base = library_dir().resolve() + path = (base / stored_name).resolve() + try: + path.relative_to(base) + except ValueError: + return None + return path if path.is_file() else None + + +# --- Creating ---------------------------------------------------------------- +def store_upload( + db: DBSession, *, owner: User, payload: bytes, filename: str, title: str = "" +) -> Document: + """Add an uploaded file to the library. Raises files.FileError if unusable.""" + prepared = files_service.prepare(payload, filename) + + stored_name = f"{secrets.token_hex(16)}{prepared.extension}" + (library_dir() / stored_name).write_bytes(prepared.payload) + + display = files_service.safe_display_name(filename) + document = Document( + owner_id=owner.id, + title=(title.strip() or display)[:300], + source=SOURCE_UPLOAD, + filename=display, + stored_name=stored_name, + media_type=prepared.media_type, + size_bytes=len(prepared.payload), + kind=prepared.kind, + width=prepared.width, + height=prepared.height, + extracted_text=prepared.extracted_text, + pages=prepared.pages, + truncated=prepared.truncated, + extraction_error=prepared.extraction_error, + ) + db.add(document) + db.commit() + log.info("library: stored %r (%s) for %s", document.title, document.kind, owner.email) + return document + + +def store_page(db: DBSession, *, owner: User, page: Fetched) -> Document: + """Add a fetched web page to the library. + + Saved as text rather than as the original HTML: the point of keeping it is + what it said, and the markup would have to be reduced again on every read. + """ + document = Document( + owner_id=owner.id, + title=page.title[:300] or page.url[:300], + source=SOURCE_LINK, + source_url=page.url, + filename="", + media_type="text/plain", + size_bytes=len(page.text.encode("utf-8")), + kind="text", + extracted_text=page.text, + truncated=page.truncated, + ) + db.add(document) + db.commit() + log.info("library: saved page %r for %s", document.title, owner.email) + return document + + +# --- Reading ----------------------------------------------------------------- +def visible(db: DBSession, user: User | None): + return select(Document).where(sharing.visible_to(Document, user)) + + +def get(db: DBSession, document_id: str, user: User | None) -> Document | None: + document = db.get(Document, document_id) + if document is None or not sharing.can_read(db, document, user): + return None + return document + + +def search( + db: DBSession, user: User | None, needle: str, *, limit: int = 10 +) -> list[Document]: + """Documents matching `needle` that this user may see, best match first. + + The index is searched first and the visibility filter applied to the rows + it returned. That order matters: filtering afterwards is what makes it + impossible for a hit on somebody else's document to leak, even as a count. + """ + hits = search_ids(db, INDEX, needle, limit=limit * 4) + if not hits: + return [] + + order = {hit.id: position for position, hit in enumerate(hits)} + rows = list( + db.scalars(visible(db, user).where(Document.id.in_(list(order)))) + ) + rows.sort(key=lambda document: order.get(document.id, len(order))) + return rows[:limit] + + +def snippet(document: Document) -> str: + """The part of a document a search result carries.""" + text = (document.extracted_text or "").strip() + if len(text) <= SNIPPET_CHARS: + return text + return text[:SNIPPET_CHARS].rstrip() + "…" + + +# --- Removing ---------------------------------------------------------------- +def delete(db: DBSession, document: Document) -> None: + path = stored_path(document.stored_name) + if path is not None: + path.unlink(missing_ok=True) + # Shares carry no foreign key to their resource, so nothing cascades. + sharing.forget_resource(db, document) + db.delete(document) + db.commit() diff --git a/src/lembas/services/library/fts.py b/src/lembas/services/library/fts.py new file mode 100644 index 0000000..14daeca --- /dev/null +++ b/src/lembas/services/library/fts.py @@ -0,0 +1,95 @@ +"""Querying the full-text indexes. + +One helper for all three stores. The interesting part is turning what somebody +typed into something FTS5 will accept: its MATCH syntax has operators (`AND`, +`NEAR`, `*`, `^`, `:`) and a quoting rule, so a bare question mark or an +unbalanced quote is a syntax error rather than a search that finds nothing. + +Every token is therefore quoted and the operators are dropped. That costs the +ability to type an FTS expression on purpose, which nobody was going to do, and +buys a search box that cannot be made to throw. + +Search returns ids and leaves loading to the caller, which is what keeps the +visibility filter in one place: `services.sharing.visible_to` is applied to the +row query, not here. +""" + +from __future__ import annotations + +import logging +import re +from dataclasses import dataclass + +from sqlalchemy import text +from sqlalchemy.orm import Session as DBSession + +log = logging.getLogger(__name__) + +# Anything that is not a word character or an apostrophe is a separator. Keeps +# accented letters (\w is Unicode-aware here) and loses the operators. +_TOKENS = re.compile(r"[^\W_]+(?:'[^\W_]+)*", re.UNICODE) + +MAX_TERMS = 24 + + +@dataclass(frozen=True) +class SearchHit: + id: str + rank: float + + +def _terms(needle: str) -> list[str]: + tokens = _TOKENS.findall(needle or "")[:MAX_TERMS] + # Doubling any embedded quote is the FTS5 escape; tokens cannot contain one + # after the regex above, but the rule is written out so it stays true if the + # pattern is ever loosened. + return ['"' + token.replace('"', '""') + '"' for token in tokens] + + +def fts_query(needle: str, *, operator: str = "AND") -> str: + """Turn typed text into a safe FTS5 MATCH expression.""" + terms = _terms(needle) + return f" {operator} ".join(terms) if terms else "" + + +def search_ids( + db: DBSession, index: str, needle: str, *, limit: int = 20 +) -> list[SearchHit]: + """Ids matching `needle`, best first. + + `index` is a table name from db.migrations.FTS_INDEXES and never comes from + a request -- it is interpolated because SQLite cannot parameterise an + identifier, so it must stay that way. + + Every term is required first, then any of them. AND alone is right for a + search box, where more words should narrow the result -- but the caller here + is usually a *model*, which writes "who built the west gate of Moria and + what is its password" rather than "moria gate". One word absent from the + document then loses the match entirely. Falling back to OR keeps precision + where it works and recall where it does not, and bm25 sorts the difference + out: documents matching more terms rank higher anyway. + """ + if not fts_query(needle): + return [] + + def run(query: str) -> list[SearchHit]: + try: + rows = db.execute( + text( + f"SELECT id, bm25({index}) AS rank FROM {index} " # noqa: S608 - see above + f"WHERE {index} MATCH :q ORDER BY rank LIMIT :limit" + ), + {"q": query, "limit": max(1, min(limit, 100))}, + ).fetchall() + except Exception: # noqa: BLE001 - a broken index must not break the page + log.exception("full-text search failed on %s", index) + # Rolled back because a failed statement leaves the session + # unusable: without this, one broken search turns into every later + # query in the same request failing too, which looks nothing like a + # search problem. + db.rollback() + return [] + # bm25 returns a negative number, better matches being more negative. + return [SearchHit(id=row[0], rank=float(row[1])) for row in rows] + + return run(fts_query(needle)) or run(fts_query(needle, operator="OR")) diff --git a/src/lembas/services/library/memories.py b/src/lembas/services/library/memories.py new file mode 100644 index 0000000..6dcec30 --- /dev/null +++ b/src/lembas/services/library/memories.py @@ -0,0 +1,119 @@ +"""Memory: short facts, in front of the model on every turn. + +The whole design follows from being injected rather than searched. + +* Each record is **capped short**, because every one of them costs tokens on + every request forever. A tool that writes an essay gets it trimmed and is + told so, rather than the write failing -- the model can then decide to put + the long version in a note. +* There is a **budget** for the block as a whole. Past it the oldest are left + out rather than the request growing without limit; the user can see the whole + list in their settings and prune it. +* There is **no search tool**. Searching something the model is already looking + at is a round trip for nothing. +* They are **not shareable**. A record about a person is not content to hand + round, and nobody asked to share their memories with a group. +""" + +from __future__ import annotations + +import logging + +from sqlalchemy import func, select +from sqlalchemy.orm import Session as DBSession + +from lembas.db.models import AUTHOR_MODEL, AUTHOR_USER, Memory, User + +log = logging.getLogger(__name__) + +# One fact, not a paragraph. Long enough for "prefers metric units and a 24-hour +# clock", short enough that fifty of them are still affordable. +MAX_MEMORY_CHARS = 400 + +# Ceiling on the injected block. Reached, the oldest records drop out of the +# prompt -- they are still listed in settings, so nothing disappears silently. +MAX_TOTAL_CHARS = 4000 + +# A hard stop on how many can exist, so an enthusiastic model cannot fill a +# database with variations on one fact. +MAX_RECORDS = 200 + + +def all_for(db: DBSession, user: User | None) -> list[Memory]: + if user is None: + return [] + return list( + db.scalars( + select(Memory).where(Memory.owner_id == user.id).order_by(Memory.created_at) + ) + ) + + +def get(db: DBSession, memory_id: str, user: User | None) -> Memory | None: + memory = db.get(Memory, memory_id) + if memory is None or user is None or memory.owner_id != user.id: + return None + return memory + + +def add(db: DBSession, *, owner: User, content: str, author: str = AUTHOR_MODEL) -> Memory: + """Record a fact. Raises ValueError when there is no room or nothing to say.""" + content = " ".join((content or "").split()) + if not content: + raise ValueError("A memory cannot be empty.") + + count = db.scalar( + select(func.count()).select_from(Memory).where(Memory.owner_id == owner.id) + ) + if (count or 0) >= MAX_RECORDS: + raise ValueError( + f"There are already {MAX_RECORDS} memories. Remove one first, or put " + f"this in a note instead." + ) + + memory = Memory( + owner_id=owner.id, + content=content[:MAX_MEMORY_CHARS], + author=author if author in (AUTHOR_USER, AUTHOR_MODEL) else AUTHOR_MODEL, + ) + db.add(memory) + db.commit() + return memory + + +def update(db: DBSession, memory: Memory, content: str) -> Memory: + content = " ".join((content or "").split()) + if not content: + raise ValueError("A memory cannot be empty.") + memory.content = content[:MAX_MEMORY_CHARS] + db.commit() + return memory + + +def delete(db: DBSession, memory: Memory) -> None: + db.delete(memory) + db.commit() + + +def block(db: DBSession, user: User | None) -> str: + """The memories as they appear in the prompt, within the budget. + + Oldest first, and truncation drops the *newest* -- a fact that has survived + a long time is more likely to be a standing preference than something said + once this morning. + """ + records = all_for(db, user) + if not records: + return "" + + lines: list[str] = [] + total = 0 + for memory in records: + line = f"- {memory.content}" + if total + len(line) > MAX_TOTAL_CHARS: + lines.append(f"- (…{len(records) - len(lines)} more, see your settings)") + break + lines.append(line) + total += len(line) + + return "\n".join(lines) diff --git a/src/lembas/services/library/notes.py b/src/lembas/services/library/notes.py new file mode 100644 index 0000000..9d025a8 --- /dev/null +++ b/src/lembas/services/library/notes.py @@ -0,0 +1,92 @@ +"""Notes: what the model wrote down, and what a person wrote for it. + +Longer and more specific than a memory, and not injected. A dozen notes would +fill a context window on their own, so the model searches for the one it needs +-- which is also why a note has a title worth reading: it is what a search +result shows. +""" + +from __future__ import annotations + +import logging + +from sqlalchemy import select +from sqlalchemy.orm import Session as DBSession + +from lembas.db.models import AUTHOR_MODEL, AUTHOR_USER, Note, User +from lembas.services import sharing +from lembas.services.library.fts import search_ids + +log = logging.getLogger(__name__) + +INDEX = "notes_fts" + +MAX_TITLE_CHARS = 300 +MAX_BODY_CHARS = 40_000 +SNIPPET_CHARS = 800 + + +def visible(db: DBSession, user: User | None): + return select(Note).where(sharing.visible_to(Note, user)) + + +def get(db: DBSession, note_id: str, user: User | None) -> Note | None: + note = db.get(Note, note_id) + if note is None or not sharing.can_read(db, note, user): + return None + return note + + +def recent(db: DBSession, user: User | None, *, limit: int = 20) -> list[Note]: + return list( + db.scalars(visible(db, user).order_by(Note.updated_at.desc()).limit(limit)) + ) + + +def search(db: DBSession, user: User | None, needle: str, *, limit: int = 10) -> list[Note]: + """Notes matching `needle` that this user may see, best match first.""" + hits = search_ids(db, INDEX, needle, limit=limit * 4) + if not hits: + return [] + order = {hit.id: position for position, hit in enumerate(hits)} + rows = list(db.scalars(visible(db, user).where(Note.id.in_(list(order))))) + rows.sort(key=lambda note: order.get(note.id, len(order))) + return rows[:limit] + + +def create( + db: DBSession, *, owner: User, title: str, body: str, author: str = AUTHOR_USER +) -> Note: + note = Note( + owner_id=owner.id, + title=(title.strip() or "Untitled")[:MAX_TITLE_CHARS], + body=body.strip()[:MAX_BODY_CHARS], + author=author if author in (AUTHOR_USER, AUTHOR_MODEL) else AUTHOR_USER, + ) + db.add(note) + db.commit() + return note + + +def update(db: DBSession, note: Note, *, title: str | None = None, body: str | None = None) -> Note: + """Change a note. Absent arguments are left alone, which is what lets a tool + edit only the body without having to send the title back.""" + if title is not None and title.strip(): + note.title = title.strip()[:MAX_TITLE_CHARS] + if body is not None: + note.body = body.strip()[:MAX_BODY_CHARS] + db.commit() + return note + + +def delete(db: DBSession, note: Note) -> None: + sharing.forget_resource(db, note) + db.delete(note) + db.commit() + + +def snippet(note: Note) -> str: + text = (note.body or "").strip() + if len(text) <= SNIPPET_CHARS: + return text + return text[:SNIPPET_CHARS].rstrip() + "…" diff --git a/src/lembas/services/library/skills.py b/src/lembas/services/library/skills.py new file mode 100644 index 0000000..22c99dd --- /dev/null +++ b/src/lembas/services/library/skills.py @@ -0,0 +1,207 @@ +"""Skills: named instructions the model can choose to follow. + +Two fields carry the design. + +`description` is what gets injected -- one line per skill, for every skill -- +and is therefore the only thing the model has to go on when deciding whether a +skill is relevant. A description that does not say *when* to use the skill makes +it invisible in practice. + +`body` is fetched only when the model decides to use it. That split is what +makes a hundred skills affordable: the index costs a line each, the instructions +cost nothing until wanted. + +**A model may rewrite its own skills**, which is the point -- it is how it +learns a procedure once instead of being told every time. The safety story is +not a gate but a record: every write snapshots what was there first, so a change +can be read and undone. A skill written after reading a hostile web page is a +real risk, and the honest mitigation is that it is visible, attributed and +revertible rather than that it was somehow prevented. +""" + +from __future__ import annotations + +import logging +import re + +from sqlalchemy import select +from sqlalchemy.orm import Session as DBSession + +from lembas.db.models import AUTHOR_MODEL, AUTHOR_USER, Skill, SkillRevision, User +from lembas.services import sharing +from lembas.services.library.fts import search_ids + +log = logging.getLogger(__name__) + +INDEX = "skills_fts" + +# A name the model can quote back without getting it wrong. +SKILL_NAME_PATTERN = re.compile(r"^[a-z0-9][a-z0-9-]{1,60}$") + +MAX_DESCRIPTION_CHARS = 400 +MAX_BODY_CHARS = 20_000 + +# The index goes into every request, so it has a ceiling like memory does. +MAX_INDEX_SKILLS = 60 + + +class SkillError(Exception): + """A rejected skill write, with a message fit for the model or the user.""" + + +def slugify(name: str) -> str: + cleaned = re.sub(r"[^a-z0-9]+", "-", (name or "").strip().lower()).strip("-") + return cleaned[:60] + + +def visible(db: DBSession, user: User | None): + return select(Skill).where(sharing.visible_to(Skill, user)) + + +def get(db: DBSession, skill_id: str, user: User | None) -> Skill | None: + skill = db.get(Skill, skill_id) + if skill is None or not sharing.can_read(db, skill, user): + return None + return skill + + +def by_name(db: DBSession, name: str, user: User | None) -> Skill | None: + """Look one up the way the model refers to it.""" + if user is None: + return None + return db.scalar(visible(db, user).where(Skill.name == slugify(name))) + + +def enabled_for(db: DBSession, user: User | None) -> list[Skill]: + """Skills that should appear in the index, oldest first for a stable order.""" + if user is None: + return [] + return list( + db.scalars( + visible(db, user) + .where(Skill.enabled.is_(True)) + .order_by(Skill.name) + .limit(MAX_INDEX_SKILLS) + ) + ) + + +def search(db: DBSession, user: User | None, needle: str, *, limit: int = 10) -> list[Skill]: + hits = search_ids(db, INDEX, needle, limit=limit * 4) + if not hits: + return [] + order = {hit.id: position for position, hit in enumerate(hits)} + rows = list(db.scalars(visible(db, user).where(Skill.id.in_(list(order))))) + rows.sort(key=lambda skill: order.get(skill.id, len(order))) + return rows[:limit] + + +def snapshot(db: DBSession, skill: Skill, *, author: str, note: str = "") -> SkillRevision: + """Record what a skill looked like before it is changed.""" + revision = SkillRevision( + skill_id=skill.id, + description=skill.description, + body=skill.body, + author=author, + note=note[:200], + ) + db.add(revision) + return revision + + +def create( + db: DBSession, + *, + owner: User, + name: str, + description: str, + body: str, + author: str = AUTHOR_USER, +) -> Skill: + slug = slugify(name) + if not SKILL_NAME_PATTERN.match(slug): + raise SkillError( + "A skill name must be two or more letters, numbers or hyphens, " + "such as 'weekly-report'." + ) + if by_name(db, slug, owner) is not None: + raise SkillError(f"A skill called {slug!r} already exists. Edit it instead.") + if not description.strip(): + raise SkillError( + "A skill needs a description saying when to use it — it is the only " + "thing shown until the skill is opened." + ) + + skill = Skill( + owner_id=owner.id, + name=slug, + description=description.strip()[:MAX_DESCRIPTION_CHARS], + body=body.strip()[:MAX_BODY_CHARS], + author=author if author in (AUTHOR_USER, AUTHOR_MODEL) else AUTHOR_USER, + ) + db.add(skill) + db.commit() + log.info("skill %r created by %s", slug, author) + return skill + + +def update( + db: DBSession, + skill: Skill, + *, + description: str | None = None, + body: str | None = None, + enabled: bool | None = None, + author: str = AUTHOR_USER, + note: str = "", +) -> Skill: + """Change a skill, keeping what it was. + + The snapshot happens before the change and in the same transaction, so + there is no window where a skill has been rewritten with no record of what + it used to say. + """ + changing = (description is not None and description.strip() != skill.description) or ( + body is not None and body.strip() != skill.body + ) + if changing: + snapshot(db, skill, author=author, note=note) + + if description is not None and description.strip(): + skill.description = description.strip()[:MAX_DESCRIPTION_CHARS] + if body is not None: + skill.body = body.strip()[:MAX_BODY_CHARS] + if enabled is not None: + skill.enabled = enabled + if changing: + skill.author = author if author in (AUTHOR_USER, AUTHOR_MODEL) else skill.author + + db.commit() + return skill + + +def revert(db: DBSession, skill: Skill, revision: SkillRevision, *, author: str) -> Skill: + """Put a skill back to an earlier revision. + + The revert is itself a change, so the current state is snapshotted first -- + going back is undoable too. + """ + snapshot(db, skill, author=author, note="before revert") + skill.description = revision.description + skill.body = revision.body + db.commit() + return skill + + +def delete(db: DBSession, skill: Skill) -> None: + sharing.forget_resource(db, skill) + db.delete(skill) + db.commit() + + +def index_block(db: DBSession, user: User | None) -> str: + """The one-line-per-skill listing that goes into the prompt.""" + skills = enabled_for(db, user) + if not skills: + return "" + return "\n".join(f"- {skill.name}: {skill.description}" for skill in skills) diff --git a/src/lembas/services/settings_store.py b/src/lembas/services/settings_store.py index e292954..c840a1d 100644 --- a/src/lembas/services/settings_store.py +++ b/src/lembas/services/settings_store.py @@ -76,6 +76,11 @@ def _search_defaults() -> dict[str, Any]: "firecrawl_base_url": "https://api.firecrawl.dev", "firecrawl_api_key_encrypted": "", "timeout": 20.0, + # Whether saving a link may reach addresses on this machine or this + # network. Off, because a server that fetches any URL it is handed can + # be pointed at a router's admin page or at LLeMbas itself, and the URL + # can come from a model. See services/fetch.py. + "allow_private_fetch": False, } diff --git a/src/lembas/services/sharing.py b/src/lembas/services/sharing.py new file mode 100644 index 0000000..9da65e4 --- /dev/null +++ b/src/lembas/services/sharing.py @@ -0,0 +1,205 @@ +"""Who may see a document, a note or a skill. + +One rule, in one place, for all three: you can see a resource if you own it, if +it was shared with you by name, or if it was shared with a group you are in. + +Everything that lists or searches a library store goes through `visible_to`. +Writing the same condition into each query would work right up until one of +them was written slightly differently, and the way that failure shows up is +somebody reading somebody else's notes. + +**Administrators are not exempt.** They are elsewhere in this codebase -- +`security.permissions.resolve` hands an admin every permission -- and that is +right for configuration, because an admin can grant themselves those two clicks +away. This is a different thing. Nobody made these records available to anyone, +and being able to reach a database is not the same as being invited. +""" + +from __future__ import annotations + +import logging +from typing import Any + +from sqlalchemy import ColumnElement, delete, or_, select +from sqlalchemy.orm import Session as DBSession + +from lembas.db.models import ( + PRINCIPAL_GROUP, + PRINCIPAL_USER, + RESOURCE_DOCUMENT, + RESOURCE_NOTE, + RESOURCE_SKILL, + Document, + Note, + Share, + Skill, + User, +) + +log = logging.getLogger(__name__) + +# The mapping between a model class and the string stored in Share. Kept here +# so no caller has to remember which literal goes with which table. +RESOURCE_TYPES: dict[Any, str] = { + Document: RESOURCE_DOCUMENT, + Note: RESOURCE_NOTE, + Skill: RESOURCE_SKILL, +} + + +def resource_type(model: Any) -> str: + kind = RESOURCE_TYPES.get(model if isinstance(model, type) else type(model)) + if kind is None: + raise ValueError(f"{model!r} is not a shareable resource") + return kind + + +def principal_ids(user: User | None) -> tuple[list[str], list[str]]: + """The ids a share could name to reach this user: themselves, their groups.""" + if user is None: + return [], [] + return [user.id], [group.id for group in user.groups] + + +def visible_to(model: Any, user: User | None) -> ColumnElement[bool]: + """A WHERE clause selecting the rows of `model` this user may see. + + Returned as a condition rather than a query so callers can add their own + filtering, ordering and pagination without this module knowing about any of + it. + """ + if user is None: + # Signed out sees nothing. Not an empty library -- no library. + return model.id.is_(None) + + users, groups = principal_ids(user) + shared = select(Share.resource_id).where( + Share.resource_type == resource_type(model), + or_( + (Share.principal_type == PRINCIPAL_USER) & Share.principal_id.in_(users), + (Share.principal_type == PRINCIPAL_GROUP) & Share.principal_id.in_(groups or [""]), + ), + ) + return or_(model.owner_id == user.id, model.id.in_(shared)) + + +def owned_by(model: Any, user: User | None) -> ColumnElement[bool]: + """Rows this user may *change*. + + Sharing grants reading, never writing. Two people editing one note with no + history and no merge is worse than the inconvenience of copying it. + """ + if user is None: + return model.id.is_(None) + return model.owner_id == user.id + + +def can_read(db: DBSession, resource: Any, user: User | None) -> bool: + if user is None or resource is None: + return False + if resource.owner_id == user.id: + return True + users, groups = principal_ids(user) + found = db.scalar( + select(Share.id).where( + Share.resource_type == resource_type(resource), + Share.resource_id == resource.id, + or_( + (Share.principal_type == PRINCIPAL_USER) & Share.principal_id.in_(users), + (Share.principal_type == PRINCIPAL_GROUP) + & Share.principal_id.in_(groups or [""]), + ), + ) + ) + return found is not None + + +def can_write(resource: Any, user: User | None) -> bool: + return user is not None and resource is not None and resource.owner_id == user.id + + +# --- Managing grants --------------------------------------------------------- +def grants_for(db: DBSession, resource: Any) -> list[Share]: + return list( + db.scalars( + select(Share).where( + Share.resource_type == resource_type(resource), + Share.resource_id == resource.id, + ) + ) + ) + + +def set_grants( + db: DBSession, + resource: Any, + *, + user_ids: list[str], + group_ids: list[str], +) -> None: + """Replace a resource's shares with exactly these principals.""" + kind = resource_type(resource) + db.execute( + delete(Share).where(Share.resource_type == kind, Share.resource_id == resource.id) + ) + + wanted = [(PRINCIPAL_USER, i) for i in dict.fromkeys(user_ids) if i] + [ + (PRINCIPAL_GROUP, i) for i in dict.fromkeys(group_ids) if i + ] + for principal_type, principal_id in wanted: + # Sharing with yourself is not wrong, just meaningless -- you own it. + if principal_type == PRINCIPAL_USER and principal_id == resource.owner_id: + continue + db.add( + Share( + resource_type=kind, + resource_id=resource.id, + principal_type=principal_type, + principal_id=principal_id, + ) + ) + db.commit() + + +def forget_resource(db: DBSession, resource: Any) -> None: + """Drop every share of a resource that is being deleted. + + Shares carry no foreign key to their resource -- one column pointing at + three tables cannot have one -- so nothing cascades and this has to be + called explicitly. + """ + db.execute( + delete(Share).where( + Share.resource_type == resource_type(resource), + Share.resource_id == resource.id, + ) + ) + + +def forget_principal(db: DBSession, principal_type: str, principal_id: str) -> int: + """Drop every share naming a user or group that has been deleted. + + Same reason as above: no foreign key, so nothing cascades. Called when an + account or a group goes; a stale row would otherwise grant access to + whoever next received that id, which is not a risk worth carrying for the + sake of a tidy delete. + """ + result = db.execute( + delete(Share).where( + Share.principal_type == principal_type, Share.principal_id == principal_id + ) + ) + return result.rowcount or 0 + + +__all__ = [ + "can_read", + "can_write", + "forget_principal", + "forget_resource", + "grants_for", + "owned_by", + "resource_type", + "set_grants", + "visible_to", +] diff --git a/src/lembas/services/tools.py b/src/lembas/services/tools.py index fddc78a..104ee38 100644 --- a/src/lembas/services/tools.py +++ b/src/lembas/services/tools.py @@ -1,19 +1,24 @@ """Tools a model may call while it answers. -One tool so far -- web search -- but the shape is the point: a registry of -named callables with a JSON schema each, offered to the endpoint and executed -here when it asks. Built-in tools, MCP servers and agentic execution all plug -in at the same place. +A registry of named callables with a JSON schema each: offered to the endpoint, +executed here when it asks. MCP servers and agentic execution plug in at the +same place, which is why the registry is keyed and grouped rather than being a +handful of if-statements. -Two things gate whether a tool is offered at all: +Three things gate whether a tool is offered: -* the administrator has configured and enabled it, and -* the chat's model is marked as supporting tools. +* the instance is configured for it (web search has a provider, and so on), +* the reader has the permission, and +* the chat's model is marked as having that tool. -The second is not optional politeness. Sending a ``tools`` array to an endpoint +The last is not optional politeness. Sending a ``tools`` array to an endpoint that does not implement tool calling fails the entire request, exactly the way -sending image parts to a model without vision does -- and for the same reason, -the capability flag on the model is what decides. +sending image parts to a model without vision does. + +Tools that *write* -- notes, memories, skills -- need a database session and a +user, and they run inside a background generation that outlives the request. So +they are handed a `ToolContext` carrying an owner id rather than a live session, +and open their own scope, the same way `services.generation` does. """ from __future__ import annotations @@ -26,9 +31,14 @@ from typing import Any from sqlalchemy.orm import Session as DBSession -from lembas.db.models import Chat, User +from lembas.db.models import AUTHOR_MODEL, Chat, User +from lembas.db.session import session_scope from lembas.services import search as search_service from lembas.services import settings_store +from lembas.services.library import documents as documents_service +from lembas.services.library import memories as memories_service +from lembas.services.library import notes as notes_service +from lembas.services.library import skills as skills_service from lembas.services.search.base import SearchError log = logging.getLogger(__name__) @@ -39,34 +49,30 @@ log = logging.getLogger(__name__) # out, and each round costs a full request. MAX_ROUNDS = 3 -WEB_SEARCH = "web_search" +# Tool families, matching the per-model capability flags and the permission +# keys. The three names differ by prefix only, which is deliberate: adding a +# family means adding one entry here and one permission. +FAMILY_SEARCH = "web_search" +FAMILY_KNOWLEDGE = "knowledge" +FAMILY_NOTES = "notes" +FAMILY_MEMORY = "memory" +FAMILY_SKILLS = "skills" -WEB_SEARCH_SCHEMA: dict[str, Any] = { - "type": "function", - "function": { - "name": WEB_SEARCH, - "description": ( - "Search the web for current information. Use this when the answer " - "depends on recent events, on facts you are unsure of, or on " - "anything that may have changed since your training data. Returns " - "a numbered list of results with titles, URLs and short extracts." - ), - "parameters": { - "type": "object", - "properties": { - "query": { - "type": "string", - "description": "The search terms. Keep them short and specific.", - }, - "max_results": { - "type": "integer", - "description": "How many results to return. Defaults to the site setting.", - }, - }, - "required": ["query"], - }, - }, -} +FAMILIES = (FAMILY_SEARCH, FAMILY_KNOWLEDGE, FAMILY_NOTES, FAMILY_MEMORY, FAMILY_SKILLS) + + +@dataclass +class ToolContext: + """What a tool needs to do its work, without holding a session open. + + `owner_id` rather than a User for the same reason `Endpoint` is a frozen + snapshot rather than a Connection: a generation outlives the request that + started it, and a detached SQLAlchemy instance is a trap. + """ + + owner_id: str + search_config: dict[str, Any] = field(default_factory=dict) + allow_private_fetch: bool = False @dataclass @@ -74,7 +80,7 @@ class ToolOutcome: """What running a tool produced, for the model and for the reader. The two are deliberately different. `content` is the flat text the model - reads back; `event` is what the transcript shows, and keeps the results + reads back; `event` is what the transcript shows, and keeps results structured so they can be rendered as links rather than as a wall of URLs. """ @@ -82,72 +88,62 @@ class ToolOutcome: event: dict[str, Any] = field(default_factory=dict) -def enabled_tools(db: DBSession, chat: Chat, user: User | None) -> list[dict[str, Any]]: - """The tool schemas to offer for this chat, which is usually none.""" - from lembas.security import permissions - from lembas.services import chat as chat_service - - config = settings_store.search(db) - if not config.get("enabled"): - return [] - if not permissions.has(db, user, "tools.web_search"): - return [] - if not chat_service.model_supports(db, chat, "tools"): - return [] - if search_service.availability(str(config.get("provider") or "ddgs")): - # Configured but unusable -- offering a tool that will fail on every - # call is worse than not offering it. - return [] - return [WEB_SEARCH_SCHEMA] +Runner = Callable[[ToolContext, dict[str, Any]], Awaitable[ToolOutcome]] -async def run_tool(config: dict[str, Any], name: str, arguments: str) -> ToolOutcome: - """Execute one tool call. +@dataclass(frozen=True) +class ToolDef: + name: str + family: str + description: str + parameters: dict[str, Any] + run: Runner - Never raises. A tool that fails hands the model an explanation and lets it - carry on -- a failed search should produce "I could not look that up" - rather than killing the whole reply. - """ - if name != WEB_SEARCH: - return ToolOutcome( - content=f"There is no tool called {name!r}.", - event={"name": name, "status": "error", "error": "Unknown tool."}, - ) + @property + def schema(self) -> dict[str, Any]: + return { + "type": "function", + "function": { + "name": self.name, + "description": self.description, + "parameters": self.parameters, + }, + } - try: - parsed = json.loads(arguments) if arguments.strip() else {} - except json.JSONDecodeError: - # Small models emit malformed argument JSON often enough that this is a - # normal path, not an exceptional one. Treat the whole string as the - # query rather than giving up. - parsed = {"query": arguments.strip()} - if not isinstance(parsed, dict): - parsed = {"query": str(parsed)} - query = str(parsed.get("query") or "").strip() +def _object(properties: dict[str, Any], required: list[str]) -> dict[str, Any]: + return {"type": "object", "properties": properties, "required": required} + + +_STRING = {"type": "string"} + + +# --- Web search -------------------------------------------------------------- +async def _run_web_search(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + query = str(args.get("query") or "").strip() if not query: return ToolOutcome( - content="No search query was given.", - event={"name": name, "status": "error", "error": "No query was given."}, + "No search query was given.", + {"name": "web_search", "status": "error", "error": "No query was given."}, ) - limit = parsed.get("max_results") + limit = args.get("max_results") try: limit = int(limit) if limit is not None else None except (TypeError, ValueError): limit = None try: - results = await search_service.run(config, query, limit=limit) + results = await search_service.run(context.search_config, query, limit=limit) except SearchError as exc: log.info("web search failed for %r: %s", query[:60], exc.message) return ToolOutcome( - content=f"The search failed: {exc.message}", - event={"name": name, "query": query, "status": "error", "error": exc.message}, + f"The search failed: {exc.message}", + {"name": "web_search", "query": query, "status": "error", "error": exc.message}, ) event = { - "name": name, + "name": "web_search", "query": query, "status": "ok", "results": [ @@ -155,14 +151,573 @@ async def run_tool(config: dict[str, Any], name: str, arguments: str) -> ToolOut for r in results ], } - if not results: - return ToolOutcome(content=f"No results were found for {query!r}.", event=event) + return ToolOutcome(f"No results were found for {query!r}.", event) lines = [f"Search results for {query!r}:"] for index, result in enumerate(results, start=1): lines.append(f"\n[{index}] {result.title}\n{result.url}\n{result.snippet}") - return ToolOutcome(content="\n".join(lines), event=event) + return ToolOutcome("\n".join(lines), event) + + +# --- Knowledge --------------------------------------------------------------- +async def _run_knowledge_search(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + query = str(args.get("query") or "").strip() + if not query: + return ToolOutcome( + "No search terms were given.", + {"name": "knowledge_search", "status": "error", "error": "No query."}, + ) + + with session_scope() as db: + user = db.get(User, context.owner_id) + found = documents_service.search(db, user, query, limit=6) + event = { + "name": "knowledge_search", + "query": query, + "status": "ok", + "results": [ + {"title": d.title, "id": d.id, "kind": d.kind, "host": d.source_url} + for d in found + ], + } + if not found: + return ToolOutcome( + f"Nothing in the knowledge library matches {query!r}.", event + ) + + lines = [f"Knowledge library matches for {query!r}:"] + for document in found: + lines.append( + f"\n[{document.id}] {document.title}\n" + f"{documents_service.snippet(document)}" + ) + lines.append( + "\nUse knowledge_get with an id in brackets to read a document in full." + ) + return ToolOutcome("\n".join(lines), event) + + +async def _run_knowledge_get(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + document_id = str(args.get("id") or "").strip() + with session_scope() as db: + user = db.get(User, context.owner_id) + document = documents_service.get(db, document_id, user) + if document is None: + return ToolOutcome( + "There is no such document, or it is not available to you.", + {"name": "knowledge_get", "status": "error", "error": "Not found."}, + ) + event = { + "name": "knowledge_get", + "query": document.title, + "status": "ok", + "results": [{"title": document.title, "id": document.id}], + } + body = document.extracted_text or document.extraction_error or "(no text)" + return ToolOutcome(f"{document.title}\n\n{body}", event) + + +# --- Notes ------------------------------------------------------------------- +async def _run_notes_search(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + query = str(args.get("query") or "").strip() + with session_scope() as db: + user = db.get(User, context.owner_id) + found = ( + notes_service.search(db, user, query, limit=8) + if query + else notes_service.recent(db, user, limit=8) + ) + event = { + "name": "notes_search", + "query": query, + "status": "ok", + "results": [{"title": n.title, "id": n.id} for n in found], + } + if not found: + return ToolOutcome("There are no notes matching that.", event) + lines = ["Notes:"] + for note in found: + lines.append(f"\n[{note.id}] {note.title}\n{notes_service.snippet(note)}") + lines.append("\nUse notes_get with an id to read one in full.") + return ToolOutcome("\n".join(lines), event) + + +async def _run_notes_get(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + with session_scope() as db: + user = db.get(User, context.owner_id) + note = notes_service.get(db, str(args.get("id") or ""), user) + if note is None: + return ToolOutcome( + "There is no such note, or it is not available to you.", + {"name": "notes_get", "status": "error", "error": "Not found."}, + ) + return ToolOutcome( + f"{note.title}\n\n{note.body}", + { + "name": "notes_get", + "query": note.title, + "status": "ok", + "results": [{"title": note.title, "id": note.id}], + }, + ) + + +async def _run_notes_create(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + title = str(args.get("title") or "").strip() + body = str(args.get("body") or "").strip() + if not body: + return ToolOutcome( + "A note needs a body.", + {"name": "notes_create", "status": "error", "error": "Empty body."}, + ) + with session_scope() as db: + user = db.get(User, context.owner_id) + note = notes_service.create( + db, owner=user, title=title, body=body, author=AUTHOR_MODEL + ) + return ToolOutcome( + f"Saved note {note.id} — {note.title!r}.", + { + "name": "notes_create", + "query": note.title, + "status": "ok", + "results": [{"title": note.title, "id": note.id}], + }, + ) + + +async def _run_notes_edit(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + with session_scope() as db: + user = db.get(User, context.owner_id) + note = notes_service.get(db, str(args.get("id") or ""), user) + if note is None or note.owner_id != context.owner_id: + return ToolOutcome( + "There is no such note, or it belongs to someone else. A note " + "shared with you can be read but not changed.", + {"name": "notes_edit", "status": "error", "error": "Not writable."}, + ) + notes_service.update( + db, + note, + title=args.get("title"), + body=args.get("body"), + ) + return ToolOutcome( + f"Updated note {note.id}.", + { + "name": "notes_edit", + "query": note.title, + "status": "ok", + "results": [{"title": note.title, "id": note.id}], + }, + ) + + +async def _run_notes_delete(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + with session_scope() as db: + user = db.get(User, context.owner_id) + note = notes_service.get(db, str(args.get("id") or ""), user) + if note is None or note.owner_id != context.owner_id: + return ToolOutcome( + "There is no such note, or it belongs to someone else.", + {"name": "notes_delete", "status": "error", "error": "Not writable."}, + ) + title = note.title + notes_service.delete(db, note) + return ToolOutcome( + f"Deleted note {title!r}.", + {"name": "notes_delete", "query": title, "status": "ok", "results": []}, + ) + + +# --- Memory ------------------------------------------------------------------ +async def _run_memory_add(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + content = str(args.get("content") or "").strip() + with session_scope() as db: + user = db.get(User, context.owner_id) + try: + memory = memories_service.add( + db, owner=user, content=content, author=AUTHOR_MODEL + ) + except ValueError as exc: + return ToolOutcome( + str(exc), {"name": "memory_add", "status": "error", "error": str(exc)} + ) + + note = "" + if len(content) > memories_service.MAX_MEMORY_CHARS: + # Trimmed rather than refused, with the model told so -- it can then + # decide to put the long version in a note. + note = ( + f" It was shortened to {memories_service.MAX_MEMORY_CHARS} characters; " + f"use notes for anything longer." + ) + return ToolOutcome( + f"Remembered: {memory.content}{note}", + { + "name": "memory_add", + "query": memory.content, + "status": "ok", + "results": [], + }, + ) + + +async def _run_memory_forget(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + wanted = str(args.get("content") or "").strip().lower() + with session_scope() as db: + user = db.get(User, context.owner_id) + records = memories_service.all_for(db, user) + match = next((m for m in records if wanted and wanted in m.content.lower()), None) + if match is None: + return ToolOutcome( + "No memory matches that. The full list is in the prompt already.", + {"name": "memory_forget", "status": "error", "error": "No match."}, + ) + content = match.content + memories_service.delete(db, match) + return ToolOutcome( + f"Forgotten: {content}", + {"name": "memory_forget", "query": content, "status": "ok", "results": []}, + ) + + +# --- Skills ------------------------------------------------------------------ +async def _run_skill_get(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + name = str(args.get("name") or "").strip() + with session_scope() as db: + user = db.get(User, context.owner_id) + skill = skills_service.by_name(db, name, user) + if skill is None: + return ToolOutcome( + f"There is no skill called {name!r}.", + {"name": "skill_get", "status": "error", "error": "Not found."}, + ) + return ToolOutcome( + f"Skill {skill.name}: {skill.description}\n\n{skill.body}", + { + "name": "skill_get", + "query": skill.name, + "status": "ok", + "results": [{"title": skill.name, "id": skill.id}], + }, + ) + + +async def _run_skill_create(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + with session_scope() as db: + user = db.get(User, context.owner_id) + try: + skill = skills_service.create( + db, + owner=user, + name=str(args.get("name") or ""), + description=str(args.get("description") or ""), + body=str(args.get("body") or ""), + author=AUTHOR_MODEL, + ) + except skills_service.SkillError as exc: + return ToolOutcome( + str(exc), {"name": "skill_create", "status": "error", "error": str(exc)} + ) + return ToolOutcome( + f"Created skill {skill.name!r}.", + { + "name": "skill_create", + "query": skill.name, + "status": "ok", + "results": [{"title": skill.name, "id": skill.id}], + }, + ) + + +async def _run_skill_edit(context: ToolContext, args: dict[str, Any]) -> ToolOutcome: + with session_scope() as db: + user = db.get(User, context.owner_id) + skill = skills_service.by_name(db, str(args.get("name") or ""), user) + if skill is None or skill.owner_id != context.owner_id: + return ToolOutcome( + "There is no such skill, or it belongs to someone else.", + {"name": "skill_edit", "status": "error", "error": "Not writable."}, + ) + skills_service.update( + db, + skill, + description=args.get("description"), + body=args.get("body"), + author=AUTHOR_MODEL, + note=str(args.get("reason") or "")[:200], + ) + return ToolOutcome( + f"Updated skill {skill.name!r}. The previous version was kept and can " + f"be restored.", + { + "name": "skill_edit", + "query": skill.name, + "status": "ok", + "results": [{"title": skill.name, "id": skill.id}], + }, + ) + + +# --- The registry ------------------------------------------------------------ +REGISTRY: dict[str, ToolDef] = { + tool.name: tool + for tool in ( + ToolDef( + name="web_search", + family=FAMILY_SEARCH, + description=( + "Search the web for current information. Use this when the answer " + "depends on recent events, on facts you are unsure of, or on " + "anything that may have changed since your training data. Returns " + "a numbered list of results with titles, URLs and short extracts." + ), + parameters=_object( + { + "query": { + "type": "string", + "description": "The search terms. Keep them short and specific.", + }, + "max_results": { + "type": "integer", + "description": "How many results to return.", + }, + }, + ["query"], + ), + run=_run_web_search, + ), + ToolDef( + name="knowledge_search", + family=FAMILY_KNOWLEDGE, + description=( + "Search the user's own collected documents, files and saved web " + "pages. Use this before searching the web when the question is " + "about their material rather than about the world." + ), + parameters=_object( + {"query": {**_STRING, "description": "Words likely to appear in the document."}}, + ["query"], + ), + run=_run_knowledge_search, + ), + ToolDef( + name="knowledge_get", + family=FAMILY_KNOWLEDGE, + description="Read one knowledge document in full, by the id a search returned.", + parameters=_object({"id": _STRING}, ["id"]), + run=_run_knowledge_get, + ), + ToolDef( + name="notes_search", + family=FAMILY_NOTES, + description=( + "Search your notes. These are things you or the user wrote down in " + "earlier conversations. With no query, returns the most recent." + ), + parameters=_object({"query": _STRING}, []), + run=_run_notes_search, + ), + ToolDef( + name="notes_get", + family=FAMILY_NOTES, + description="Read one note in full, by the id a search returned.", + parameters=_object({"id": _STRING}, ["id"]), + run=_run_notes_get, + ), + ToolDef( + name="notes_create", + family=FAMILY_NOTES, + description=( + "Write a note. Use this for something worth having in a later " + "conversation that is too long or too detailed for a memory: a " + "procedure, a summary, a set of preferences with reasons." + ), + parameters=_object( + {"title": _STRING, "body": {**_STRING, "description": "Markdown."}}, + ["title", "body"], + ), + run=_run_notes_create, + ), + ToolDef( + name="notes_edit", + family=FAMILY_NOTES, + description="Change a note you can write to. Omit a field to leave it alone.", + parameters=_object({"id": _STRING, "title": _STRING, "body": _STRING}, ["id"]), + run=_run_notes_edit, + ), + ToolDef( + name="notes_delete", + family=FAMILY_NOTES, + description="Delete a note that is no longer true or useful.", + parameters=_object({"id": _STRING}, ["id"]), + run=_run_notes_delete, + ), + ToolDef( + name="memory_add", + family=FAMILY_MEMORY, + description=( + "Remember one short, durable fact about the user — a preference, a " + "constraint, how they like to be addressed. You are shown every " + "memory on every turn, so keep them few and short, and never store " + "passwords, keys or anything else secret." + ), + parameters=_object( + {"content": {**_STRING, "description": "One fact, in one sentence."}}, + ["content"], + ), + run=_run_memory_add, + ), + ToolDef( + name="memory_forget", + family=FAMILY_MEMORY, + description=( + "Remove a memory that has become wrong. Give enough of its text to " + "identify it." + ), + parameters=_object({"content": _STRING}, ["content"]), + run=_run_memory_forget, + ), + ToolDef( + name="skill_get", + family=FAMILY_SKILLS, + description=( + "Read the full instructions for one of the skills listed in your " + "prompt. Do this before following a skill — the list gives only its " + "name and what it is for." + ), + parameters=_object({"name": _STRING}, ["name"]), + run=_run_skill_get, + ), + ToolDef( + name="skill_create", + family=FAMILY_SKILLS, + description=( + "Write a new skill: a reusable procedure for a task you expect to be " + "asked again. The description must say when to use it, since that is " + "all you will see next time." + ), + parameters=_object( + { + "name": {**_STRING, "description": "Short slug, e.g. 'weekly-report'."}, + "description": {**_STRING, "description": "When to use this skill."}, + "body": {**_STRING, "description": "The instructions, in Markdown."}, + }, + ["name", "description", "body"], + ), + run=_run_skill_create, + ), + ToolDef( + name="skill_edit", + family=FAMILY_SKILLS, + description=( + "Improve one of your skills. The previous version is kept and can be " + "restored, so say why you changed it." + ), + parameters=_object( + { + "name": _STRING, + "description": _STRING, + "body": _STRING, + "reason": {**_STRING, "description": "Why the change was made."}, + }, + ["name"], + ), + run=_run_skill_edit, + ), + ) +} + + +def _family_allowed( + family: str, *, config: dict, capabilities: dict, allowed: dict +) -> bool: + """Whether one family is on for this chat. + + A model configured before the per-tool flags existed has no `tool_*` keys. + Absent counts as on when `tools` is on, so an upgrade does not silently take + web search away from every model already set up for it. + """ + default = bool(capabilities.get("tools")) + if not capabilities.get(f"tool_{family}", default): + return False + + if family == FAMILY_SEARCH: + return bool( + allowed.get("tools.web_search") + and config.get("enabled") + and not search_service.availability(str(config.get("provider") or "ddgs")) + ) + return bool(allowed.get(f"tools.{family}") and allowed.get("library.use")) + + +def enabled_tools(db: DBSession, chat: Chat, user: User | None) -> list[dict[str, Any]]: + """The tool schemas to offer for this chat.""" + from lembas.security import permissions + from lembas.services import chat as chat_service + + capabilities = {} + model = chat_service.model_for(db, chat) + if model is not None: + capabilities = model.capabilities_json or {} + + if not capabilities.get("tools"): + return [] + + allowed = permissions.resolve(db, user) + config = settings_store.search(db) + + families = { + family + for family in FAMILIES + if _family_allowed(family, config=config, capabilities=capabilities, allowed=allowed) + } + return [tool.schema for tool in REGISTRY.values() if tool.family in families] + + +def context_for(db: DBSession, user: User | None) -> ToolContext: + """The snapshot a running tool needs, taken while the session is open.""" + return ToolContext( + owner_id=user.id if user else "", + search_config=settings_store.search(db), + ) + + +async def run_tool(context: ToolContext, name: str, arguments: str) -> ToolOutcome: + """Execute one tool call. + + Never raises. A tool that fails hands the model an explanation and lets it + carry on -- a failed lookup should produce "I could not find that" rather + than killing the whole reply. + """ + tool = REGISTRY.get(name) + if tool is None: + return ToolOutcome( + f"There is no tool called {name!r}.", + {"name": name, "status": "error", "error": "Unknown tool."}, + ) + + try: + parsed = json.loads(arguments) if arguments.strip() else {} + except json.JSONDecodeError: + # Small models emit malformed argument JSON often enough that this is a + # normal path, not an exceptional one. Treat the whole string as the + # first required argument rather than giving up. + required = tool.parameters.get("required") or ["query"] + parsed = {required[0]: arguments.strip()} + if not isinstance(parsed, dict): + parsed = {"query": str(parsed)} + + try: + return await tool.run(context, parsed) + except Exception as exc: # noqa: BLE001 - a tool must never kill the reply + log.exception("tool %s failed", name) + return ToolOutcome( + f"The {name} tool failed: {exc}", + {"name": name, "status": "error", "error": str(exc)[:200]}, + ) class ToolCallAccumulator: @@ -247,14 +802,16 @@ def tool_turn(call: dict[str, Any], content: str) -> dict[str, Any]: } -ToolRunner = Callable[..., Awaitable[ToolOutcome]] - __all__ = [ + "FAMILIES", "MAX_ROUNDS", - "WEB_SEARCH", + "REGISTRY", "ToolCallAccumulator", + "ToolContext", + "ToolDef", "ToolOutcome", "assistant_turn", + "context_for", "enabled_tools", "run_tool", "tool_turn", diff --git a/src/lembas/web/static/css/admin.css b/src/lembas/web/static/css/admin.css index 2d77d7e..9d52646 100644 --- a/src/lembas/web/static/css/admin.css +++ b/src/lembas/web/static/css/admin.css @@ -69,6 +69,10 @@ transition: color var(--transition-fast), border-color var(--transition-fast); } .tabs__tab:hover { color: var(--ink); } +/* The library's tabs are links between pages rather than radios, so the active + one is marked server-side. Same bar, same look. */ +.tabs__tab.is-active { color: var(--ink); border-bottom-color: var(--accent); } +a.tabs__tab { text-decoration: none; } .tabs__panel { display: none; } diff --git a/src/lembas/web/static/css/app.css b/src/lembas/web/static/css/app.css index 226db54..06fd3d6 100644 --- a/src/lembas/web/static/css/app.css +++ b/src/lembas/web/static/css/app.css @@ -733,6 +733,34 @@ button, input, textarea, select { box-shadow: var(--shadow-lg); overflow: hidden; } +/* The composer's attach menu opens upwards: it sits at the bottom of the + window, so a menu dropping down would be off screen. */ +.picker--up .picker__menu { + top: auto; + bottom: calc(100% + var(--sp-1)); + left: 0; + right: auto; +} +.picker__menu--compact { width: min(18rem, calc(100vw - var(--sp-8))); padding: var(--sp-1); } +.picker__option-note { + font-size: var(--text-xs); + color: var(--ink-faint); + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +/* A dialog that holds a searchable list rather than a question. */ +.dialog--wide { width: min(34rem, calc(100vw - var(--sp-6))); } +.dialog__results { + max-height: min(24rem, 50vh); + overflow-y: auto; + scrollbar-width: thin; + border: 1px solid var(--border); + border-radius: var(--radius); +} +.dialog__results .picker__list { max-height: none; } + .picker__search { padding: var(--sp-2); border-bottom: 1px solid var(--border); } .input--sm { height: var(--control-h-sm); font-size: var(--text-xs); } .picker__list { max-height: 22rem; overflow-y: auto; scrollbar-width: thin; padding: var(--sp-1); } diff --git a/src/lembas/web/static/js/app.js b/src/lembas/web/static/js/app.js index e201e23..230ec31 100644 --- a/src/lembas/web/static/js/app.js +++ b/src/lembas/web/static/js/app.js @@ -148,6 +148,126 @@ }); } + /* --- Attaching something that is not a file ---------------------------- + The composer's menu offers four things; two of them are the file picker + with a different filter, and two need a round trip. Both of those post a + form and get a chip back, exactly like an upload, so the composer does not + have to know where a chip came from. */ + function chipTarget() { + return document.getElementById("attachments"); + } + + function chatId() { + var input = document.getElementById("file-input"); + var url = (input && input.dataset.uploadUrl) || ""; + var match = url.match(/chat_id=([^&]+)/); + return match ? decodeURIComponent(match[1]) : ""; + } + + function postForChip(url, body) { + var target = chipTarget(); + if (!target) return Promise.resolve(); + return fetch(url, { method: "POST", body: body, credentials: "same-origin" }) + .then(function (response) { return response.text(); }) + .then(function (html) { + target.insertAdjacentHTML("beforeend", html); + if (window.htmx) window.htmx.process(target.lastElementChild); + }); + } + + function attachLink() { + if (!window.lembas || !window.lembas.prompt) return; + window.lembas.prompt({ + title: "Attach a web page", + message: "The page is fetched now and its text attached, so it will not " + + "change between writing this and sending it.", + placeholder: "https://example.com/article", + confirmLabel: "Fetch", + }).then(function (url) { + if (!url || !url.trim()) return; + var body = new FormData(); + body.append("url", url.trim()); + body.append("chat_id", chatId()); + return postForChip("/api/files/link", body); + }); + } + + /* The knowledge dialog re-queries the server as you type rather than + filtering in the browser: the library is searched with FTS, which is what + makes it work at five hundred documents instead of five. */ + function attachKnowledge() { + var dialog = document.createElement("dialog"); + dialog.className = "dialog dialog--wide"; + dialog.innerHTML = + '
' + + '

Attach from your library

' + + '' + + '
' + + '
' + + "
"; + document.body.appendChild(dialog); + + var search = dialog.querySelector("input"); + var results = dialog.querySelector(".dialog__results"); + var close = dialog.querySelector("button"); + + function load(query) { + fetch("/api/files/knowledge-picker?q=" + encodeURIComponent(query || ""), { + credentials: "same-origin", + }) + .then(function (response) { return response.text(); }) + .then(function (html) { results.innerHTML = html; }) + .catch(function () { results.textContent = "Could not load your library."; }); + } + + var pending = null; + search.addEventListener("input", function () { + clearTimeout(pending); + pending = setTimeout(function () { load(search.value); }, 200); + }); + + results.addEventListener("click", function (event) { + var option = event.target.closest("[data-attach-knowledge]"); + if (!option) return; + var body = new FormData(); + body.append("document_id", option.dataset.attachKnowledge); + body.append("chat_id", chatId()); + postForChip("/api/files/from-knowledge", body); + finish(); + }); + + function finish() { + dialog.close(); + setTimeout(function () { dialog.remove(); }, 200); + } + + close.addEventListener("click", finish); + dialog.addEventListener("cancel", function (event) { + event.preventDefault(); + finish(); + }); + dialog.addEventListener("click", function (event) { + if (event.target === dialog) finish(); + }); + + dialog.showModal(); + load(""); + search.focus(); + } + + document.addEventListener("click", function (event) { + var choice = event.target.closest("[data-attach]"); + if (!choice) return; + event.preventDefault(); + + var kind = choice.dataset.attach; + if (kind === "file") document.getElementById("file-input").click(); + else if (kind === "image") document.getElementById("image-input").click(); + else if (kind === "link") attachLink(); + else if (kind === "knowledge") attachKnowledge(); + }); + function setupDropzone() { var zone = document.querySelector("[data-dropzone]"); if (!zone) return; diff --git a/src/lembas/web/templates/admin/model_detail.html b/src/lembas/web/templates/admin/model_detail.html index 35660fd..31bda6c 100644 --- a/src/lembas/web/templates/admin/model_detail.html +++ b/src/lembas/web/templates/admin/model_detail.html @@ -124,10 +124,11 @@

Capabilities

- Endpoints rarely advertise these reliably, so they are your call. - reasoning shows the thinking block, - vision lets images be sent to this model, and - tools is reserved for a feature not built yet. + What this endpoint can do. Endpoints rarely advertise it reliably, so it + is your call. reasoning shows the thinking block, + vision lets images be sent, and tools is + whether a tool list may be sent at all — turn it on for a model that does + not support tool calling and every one of its replies fails.

{% for name in capabilities %} @@ -140,6 +141,35 @@
+
+

+ Built-in tools + {% if not (model.capabilities_json or {}).get("tools") %} + needs tools + {% endif %} +

+

+ What this model is given, as opposed to what it is capable of. None of it + applies unless tools is ticked above. Each one is also + subject to the instance being configured for it and to the reader's own + permissions — this only decides whether it is offered to this + model. +

+
+ {# A model configured before these existed has no tool_* keys. Ticked by + default when `tools` is on, matching what the tool registry does, so + the form shows what will actually happen rather than a row of empty + boxes that would take web search away on the next save. #} + {% for key, label in tool_capabilities %} + + {% endfor %} +
+
+

Availability

diff --git a/src/lembas/web/templates/admin/search.html b/src/lembas/web/templates/admin/search.html index c41a23d..a1877a6 100644 --- a/src/lembas/web/templates/admin/search.html +++ b/src/lembas/web/templates/admin/search.html @@ -108,6 +108,28 @@
+
+

Saving links

+

+ Applies to the composer's Link option and to anything the + model fetches: LLeMbas retrieves the page and keeps its text. +

+
+ +

+ Off by default, and worth leaving off. This server can reach your + router, your other services and LLeMbas itself; the address to fetch can + come from a model, which can be talked into things by a web page it just + read. Turn this on only if you actually want to archive pages from your + own network. +

+
+
+

Firecrawl

diff --git a/src/lembas/web/templates/chat/_composer.html b/src/lembas/web/templates/chat/_composer.html index fbd7ade..bc67d42 100644 --- a/src/lembas/web/templates/chat/_composer.html +++ b/src/lembas/web/templates/chat/_composer.html @@ -17,10 +17,16 @@
{% if can.get("files.upload") %} {# Outside the form: it is only ever read by JavaScript, and inside it would - be submitted as an empty file part on every message. #} + be submitted as an empty file part on every message. + + Two inputs rather than one whose accept attribute is rewritten: changing + accept and then calling click() in the same tick is unreliable in Safari, + and two hidden inputs cost nothing. #} + {% endif %} @@ -47,11 +53,54 @@
{% if can.get("files.upload") %} - + {# A menu rather than the file picker straight away: there are four ways + to attach something now, and only one of them is a file on disk. + Uses the same picker machinery as the model chooser -- see ui.js. #} +
+ + + +
{% endif %} +

+ Searched along with the contents, so a line here helps a model find it. +

+
+ +
+
Kind
{{ document.kind }}
+ {% if document.source_url %} +
Source
+
{{ document.source_url }}
+ {% endif %} + {% if document.pages %}
Pages
{{ document.pages }}
{% endif %} + {% if document.size_bytes %}
Size
{{ document.human_size }}
{% endif %} +
Text
+
+ {{ "{:,}".format(document.extracted_text|length) }} characters + {%- if document.truncated %} · truncated{% endif %} +
+
+ + {% if document.extraction_error %} +
+ {{ icon("warning", "alert__icon") }} {{ document.extraction_error }} +
+ {% endif %} +
+ + {% include "library/_share.html" %} + + {% if is_owner %} +
+ + {% if document.stored_name %} + Download + {% endif %} + + +
+ {% endif %} + + +{% if document.is_image and document.stored_name %} +
+

Preview

+ {{ document.title }} +
+{% elif document.extracted_text %} +
+

Text

+

What a model is given when it reads this.

+
{{ document.extracted_text }}
+
+{% endif %} +{% endblock %} diff --git a/src/lembas/web/templates/library/note_detail.html b/src/lembas/web/templates/library/note_detail.html new file mode 100644 index 0000000..b65bfde --- /dev/null +++ b/src/lembas/web/templates/library/note_detail.html @@ -0,0 +1,64 @@ +{% extends "library/_layout.html" %} +{% from "_macros.html" import icon %} +{% set section = "notes" %} + +{% block title %}{{ note.title if note else "New note" }} - LLeMbas{% endblock %} +{% block heading %}{{ note.title if note else "New note" }}{% endblock %} + +{% block library_content %} +
+ {{ icon("chevron-right", "icon--sm") }} All notes +
+ +
+
+
+ + +

+ Shown in search results, so it is what decides whether a model opens it. +

+
+
+ + +
+ {% if note and note.author == "model" %} +

+ {{ icon("sparkle", "icon--sm") }} + Written by a model. Edit it freely — it is yours. +

+ {% endif %} +
+ + {% if note %}{% include "library/_share.html" %}{% endif %} + + {% if not note or is_owner %} +
+ + {% if note %} + + + {% endif %} +
+ {% endif %} +
+ +{% if note and body_html %} +
+

Rendered

+
{{ body_html|safe }}
+
+{% endif %} +{% endblock %} diff --git a/src/lembas/web/templates/library/notes.html b/src/lembas/web/templates/library/notes.html new file mode 100644 index 0000000..2f6a39f --- /dev/null +++ b/src/lembas/web/templates/library/notes.html @@ -0,0 +1,55 @@ +{% extends "library/_layout.html" %} +{% from "_macros.html" import icon %} +{% set section = "notes" %} + +{% block title %}Notes - LLeMbas{% endblock %} +{% block heading %}Notes{% endblock %} +{% block actions %} +{{ icon("plus", "icon--sm") }} New note +{% endblock %} + +{% block library_content %} +

+ Longer things worth keeping between conversations. A model writes these itself + when it works something out, and you can edit or delete any of them — they are + yours, not its. +

+ +
+ + + {% if q %}Clear{% endif %} +
+ +{% if not notes %} +
+ {{ icon("pencil", "empty__mark") }} +

{{ "Nothing found" if q else "No notes yet" }}

+

+ {% if q %} + No note matches “{{ q }}”. + {% else %} + Give a model the notes tool and it will start keeping them, or write the + first one yourself. + {% endif %} +

+
+{% else %} +
    + {% for note in notes %} +
  • +
    + {{ note.title }} +
    {{ note.body[:160] }}{{ "…" if note.body|length > 160 }}
    +
    +
    + {% if note.author == "model" %}written by a model{% endif %} + {% if note.owner_id != user.id %}shared{% endif %} +
    +
    • + {% endfor %} +
+{% include "library/_pager.html" %} +{% endif %} +{% endblock %} diff --git a/src/lembas/web/templates/library/skill_detail.html b/src/lembas/web/templates/library/skill_detail.html new file mode 100644 index 0000000..7c6c68c --- /dev/null +++ b/src/lembas/web/templates/library/skill_detail.html @@ -0,0 +1,126 @@ +{% extends "library/_layout.html" %} +{% from "_macros.html" import icon %} +{% set section = "skills" %} + +{% block title %}{{ skill.name if skill else "New skill" }} - LLeMbas{% endblock %} +{% block heading %}{{ skill.name if skill else "New skill" }}{% endblock %} + +{% block library_content %} +
+ {{ icon("chevron-right", "icon--sm") }} All skills +
+ +
+
+ {% if not skill %} +
+ + +

+ Lowercase letters, numbers and hyphens. This is how a model asks for it, + and it cannot be changed later. +

+
+ {% endif %} + +
+ + +

+ The load-bearing field. This one line is all a model + sees until it opens the skill, so it has to say when the skill applies — + not what it does. +

+
+ +
+ + +
+ + {% if skill %} +
+ +

+ Turned off, it stays here but disappears from the list the model sees. +

+
+ {% endif %} + + {% if skill and skill.author == "model" %} +
+ {{ icon("sparkle", "alert__icon") }} +
+ A model wrote this version. +
+ Worth reading before you rely on it. A model that has just read a web + page can be talked into things by that page, and a skill persists. + Every earlier version is below. +
+
+
+ {% endif %} +
+ + {% if skill %}{% include "library/_share.html" %}{% endif %} + + {% if not skill or is_owner %} +
+ + {% if skill %} + + + {% endif %} +
+ {% endif %} +
+ +{% if skill and revisions %} +
+

History {{ revisions|length }}

+

+ What this skill said before each change. This is the whole safety story for a + model editing its own instructions: not a gate, but a record and a way back. +

+
    + {% for revision in revisions %} +
  • +
    + {{ revision.created_at.strftime("%Y-%m-%d %H:%M") }} + {% if revision.author == "model" %} + model + {% else %} + you + {% endif %} + {% if revision.note %}
    {{ revision.note }}
    {% endif %} +
    {{ revision.body[:200] }}{{ "…" if revision.body|length > 200 }}
    +
    + {% if is_owner %} +
    + +
    + {% endif %} +
    • + {% endfor %} +
+
+{% endif %} +{% endblock %} diff --git a/src/lembas/web/templates/library/skills.html b/src/lembas/web/templates/library/skills.html new file mode 100644 index 0000000..43a5858 --- /dev/null +++ b/src/lembas/web/templates/library/skills.html @@ -0,0 +1,57 @@ +{% extends "library/_layout.html" %} +{% from "_macros.html" import icon %} +{% set section = "skills" %} + +{% block title %}Skills - LLeMbas{% endblock %} +{% block heading %}Skills{% endblock %} +{% block actions %} +{{ icon("plus", "icon--sm") }} New skill +{% endblock %} + +{% block library_content %} +

+ Saved procedures. Every enabled skill's name and description are shown to the + model on each turn; it reads the instructions only when it decides one + applies. A model may write and revise its own — every version is kept, so any + change can be read and undone. +

+ +
+ + + {% if q %}Clear{% endif %} +
+ +{% if not skills %} +
+ {{ icon("sparkle", "empty__mark") }} +

{{ "Nothing found" if q else "No skills yet" }}

+

+ {% if q %} + No skill matches “{{ q }}”. + {% else %} + Write one for a task you explain often, or let a model save one after it + has worked the task out. + {% endif %} +

+
+{% else %} +
    + {% for skill in skills %} +
  • +
    + {{ skill.name }} +
    {{ skill.description }}
    +
    +
    + {% if not skill.enabled %}off{% endif %} + {% if skill.author == "model" %}written by a model{% endif %} + {% if skill.owner_id != user.id %}shared{% endif %} +
    +
    • + {% endfor %} +
+{% include "library/_pager.html" %} +{% endif %} +{% endblock %} diff --git a/src/lembas/web/templates/partials/sidebar.html b/src/lembas/web/templates/partials/sidebar.html index 1ae5d61..ddff1de 100644 --- a/src/lembas/web/templates/partials/sidebar.html +++ b/src/lembas/web/templates/partials/sidebar.html @@ -70,6 +70,13 @@
+ {% if can.get("library.use") %} + + {{ icon("archive", "icon--sm") }} + Library + + {% endif %} + {{ icon("user", "icon--sm") }} {{ user.name }} diff --git a/src/lembas/web/templates/settings.html b/src/lembas/web/templates/settings.html index cc0f2e1..4ea705a 100644 --- a/src/lembas/web/templates/settings.html +++ b/src/lembas/web/templates/settings.html @@ -40,6 +40,11 @@ {% endif %} + {% if can.get("library.use") %} + + + {% endif %} +
@@ -275,6 +280,69 @@ {% endif %} + {# --- Memory --- #} + {% if can.get("library.use") %} +
+
+

+ What models remember about you + {{ memories|length }} +

+

+ Short facts, shown to every model you talk to on every turn — so + they are few and short on purpose. Anything longer belongs in a + note. Nothing here is shared with + anyone else, ever. +

+ + {% if memories %} +
    + {% for memory in memories %} +
  • +
    + + + +
    + {% if memory.author == "model" %} + remembered + {% endif %} +
    • + {% endfor %} +
+ {% else %} +

+ Nothing yet. A model with the memory tool adds to this when you + tell it something worth keeping. +

+ {% endif %} +
+ +
+

Add one

+
+ + +
+

+ One fact per record, at most {{ memory_limit }} characters. Never + put a password or a key here — it is sent to the model with every + message. +

+
+
+ {% endif %} + {# --- Security --- #}
diff --git a/tests/conftest.py b/tests/conftest.py index 1c24e33..0a75a6a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -46,7 +46,14 @@ def fresh_database(tmp_path: Path) -> Iterator[None]: import lembas.db.models # noqa: F401 (registers the tables) + # sync_schema rather than create_all: it is what startup runs, and it also + # builds the full-text indexes, which are not SQLAlchemy models and so are + # invisible to create_all. Tests were otherwise running against a schema + # production does not have. + from lembas.db.migrations import sync_schema + Base.metadata.create_all(bind=get_engine()) + sync_schema(get_engine()) yield reset_engine() diff --git a/tests/test_fetch.py b/tests/test_fetch.py new file mode 100644 index 0000000..f9ce2c8 --- /dev/null +++ b/tests/test_fetch.py @@ -0,0 +1,194 @@ +"""Fetching a web page, and refusing to fetch the wrong ones. + +The refusals are the important half. This runs on a server that can reach the +router, the other services on the box, and LLeMbas itself — and the URL can come +from a model, which can be talked into things by a page it just read. +""" + +from __future__ import annotations + +import httpx +import pytest + +from lembas.services.fetch import FetchError, check_url, fetch, html_to_text + + +# --- What is refused --------------------------------------------------------- +@pytest.mark.parametrize( + "url", + [ + "http://127.0.0.1:8080/admin", # LLeMbas itself + "http://localhost/", # the same, by name + "http://10.0.0.1/", # the network the server is on + "http://192.168.1.1/", # a router + "http://172.16.5.4/", + "http://169.254.169.254/latest/meta-data/", # cloud metadata, i.e. credentials + "http://[::1]/", + "http://0.0.0.0/", + ], +) +def test_private_addresses_are_refused(url): + with pytest.raises(FetchError, match="private or local"): + check_url(url) + + +@pytest.mark.parametrize( + "url", ["file:///etc/passwd", "ftp://host/x", "gopher://host/", "javascript:alert(1)"] +) +def test_only_http_and_https(url): + with pytest.raises(FetchError, match="http"): + check_url(url) + + +def test_a_url_with_no_host_is_refused(): + with pytest.raises(FetchError): + check_url("http:///nothing") + + +def test_the_check_is_on_the_resolved_address(monkeypatch): + """A hostname pointing at 127.0.0.1 is the obvious way past a check that + only reads the text of the URL.""" + import socket + + monkeypatch.setattr( + socket, "getaddrinfo", lambda *a, **k: [(2, 1, 6, "", ("127.0.0.1", 80))] + ) + with pytest.raises(FetchError, match="private or local"): + check_url("http://sneaky.example.com/") + + +def test_one_private_address_among_several_is_still_refused(monkeypatch): + """A name resolving to one public and one private address must not be + usable to reach the private one.""" + import socket + + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda *a, **k: [(2, 1, 6, "", ("93.184.216.34", 80)), (2, 1, 6, "", ("10.0.0.1", 80))], + ) + with pytest.raises(FetchError, match="private or local"): + check_url("http://mixed.example.com/") + + +def test_an_administrator_can_open_it_deliberately(): + assert check_url("http://127.0.0.1:8080/", allow_private=True) + + +def test_a_public_address_passes(monkeypatch): + import socket + + monkeypatch.setattr( + socket, "getaddrinfo", lambda *a, **k: [(2, 1, 6, "", ("93.184.216.34", 80))] + ) + assert check_url("https://example.com/x") == "https://example.com/x" + + +# --- Redirects --------------------------------------------------------------- +async def test_a_redirect_to_a_private_address_is_refused(mock_http, monkeypatch): + """httpx's own following would validate the first address and then happily + land on localhost, which is why redirects are followed by hand.""" + import socket + + monkeypatch.setattr( + socket, + "getaddrinfo", + lambda host, *a, **k: [(2, 1, 6, "", ("93.184.216.34", 80))] + if host == "example.com" + else [(2, 1, 6, "", ("127.0.0.1", 80))], + ) + mock_http( + lambda _r: httpx.Response(302, headers={"location": "http://127.0.0.1:8080/admin"}) + ) + with pytest.raises(FetchError, match="private or local"): + await fetch("https://example.com/") + + +async def test_endless_redirects_end(mock_http, monkeypatch): + import socket + + monkeypatch.setattr( + socket, "getaddrinfo", lambda *a, **k: [(2, 1, 6, "", ("93.184.216.34", 80))] + ) + mock_http(lambda r: httpx.Response(302, headers={"location": str(r.url)})) + with pytest.raises(FetchError, match="redirected too many"): + await fetch("https://example.com/") + + +# --- Reducing a page --------------------------------------------------------- +def test_script_and_style_are_dropped(): + _, text = html_to_text( + "

Real text.

" + ) + assert text == "Real text." + + +def test_the_title_is_taken_and_not_repeated_in_the_body(): + title, text = html_to_text( + " A Page

Body.

" + ) + assert title == "A Page" + assert text == "Body." + + +def test_block_tags_become_line_breaks(): + """Without this the whole page arrives as one paragraph.""" + _, text = html_to_text("

One

Two

  • Three
    • ") + assert text.splitlines() == ["One", "Two", "Three"] + + +def test_entities_are_unescaped(): + _, text = html_to_text("

    Salt & pepper, 5 > 3

    ") + assert text == "Salt & pepper, 5 > 3" + + +# --- Fetching ---------------------------------------------------------------- +async def test_a_page_is_reduced_to_text(mock_http, monkeypatch): + import socket + + monkeypatch.setattr( + socket, "getaddrinfo", lambda *a, **k: [(2, 1, 6, "", ("93.184.216.34", 80))] + ) + mock_http( + lambda _r: httpx.Response( + 200, + headers={"content-type": "text/html"}, + text=( + "Mallorn" + "

    A golden tree.

    " + ), + ) + ) + page = await fetch("https://example.com/mallorn") + assert page.title == "Mallorn" + assert page.text == "A golden tree." + + +async def test_a_binary_response_is_refused_with_advice(mock_http, monkeypatch): + import socket + + monkeypatch.setattr( + socket, "getaddrinfo", lambda *a, **k: [(2, 1, 6, "", ("93.184.216.34", 80))] + ) + mock_http( + lambda _r: httpx.Response(200, headers={"content-type": "image/png"}, content=b"\x89PNG") + ) + with pytest.raises(FetchError, match="Attach it as a file"): + await fetch("https://example.com/x.png") + + +async def test_a_page_with_no_readable_text_says_so(mock_http, monkeypatch): + import socket + + monkeypatch.setattr( + socket, "getaddrinfo", lambda *a, **k: [(2, 1, 6, "", ("93.184.216.34", 80))] + ) + mock_http( + lambda _r: httpx.Response( + 200, + headers={"content-type": "text/html"}, + text="
    ", + ) + ) + with pytest.raises(FetchError, match="JavaScript"): + await fetch("https://example.com/") diff --git a/tests/test_harness.py b/tests/test_harness.py new file mode 100644 index 0000000..9e5d74e --- /dev/null +++ b/tests/test_harness.py @@ -0,0 +1,173 @@ +"""The operational preamble, and how it sits beside the authored prompt.""" + +from __future__ import annotations + +import pytest + +from lembas.db.models import Chat, Connection, Model, User +from lembas.security.passwords import hash_password +from lembas.services import chat as chat_service +from lembas.services import harness, settings_store +from lembas.services import tools as tools_service +from lembas.services.library import memories as memories_service +from lembas.services.library import skills as skills_service + + +@pytest.fixture +def owner(db): + user = User(name="Frodo", email="f@shire.test", password_hash=hash_password("x")) + db.add(user) + db.commit() + return user + + +def _tools(*names): + return [tools_service.REGISTRY[name].schema for name in names] + + +# --- Composition ------------------------------------------------------------- +def test_no_tools_means_no_harness(db, owner): + """An empty harness is worse than none: tokens that say only that there is + nothing to say.""" + assert harness.compose(db, owner, []) == "" + assert harness.compose(db, owner, None) == "" + + +def test_only_the_guidance_for_offered_tools_appears(db, owner): + text = harness.compose(db, owner, _tools("web_search")) + assert "Look things up" in text + assert "You keep notes" not in text + assert "Skills are procedures" not in text + + +def test_the_memory_block_is_included_when_memory_is_offered(db, owner): + memories_service.add(db, owner=owner, content="Prefers metric units.") + text = harness.compose(db, owner, _tools("memory_add")) + assert "What you know about this person" in text + assert "Prefers metric units." in text + + +def test_memories_are_absent_without_the_memory_tool(db, owner): + """A model not given the memory tool has no business being told them.""" + memories_service.add(db, owner=owner, content="Prefers metric units.") + text = harness.compose(db, owner, _tools("web_search")) + assert "Prefers metric units." not in text + + +def test_the_skill_index_is_names_and_descriptions_only(db, owner): + skills_service.create( + db, owner=owner, name="weekly-report", description="When asked.", body="SECRET" + ) + text = harness.compose(db, owner, _tools("skill_get")) + assert "weekly-report: When asked." in text + assert "SECRET" not in text + + +def test_an_empty_store_contributes_no_heading(db, owner): + """And the guidance must not point at a heading that is not there: telling a + model to consult an absent section is a good way to make it invent one.""" + text = harness.compose(db, owner, _tools("memory_add", "skill_get")) + assert "### What you know about this person" not in text + assert "### Skills available" not in text + assert "was remembered earlier" not in text + assert "You can remember durable facts" in text + + +def test_the_harness_is_capped(db, owner, monkeypatch): + monkeypatch.setattr(harness, "MAX_HARNESS_CHARS", 200) + for index in range(50): + skills_service.create( + db, owner=owner, name=f"skill-{index}", description="x" * 200, body="y" + ) + assert len(harness.compose(db, owner, _tools("skill_get"))) <= 202 + + +# --- Joining ----------------------------------------------------------------- +def test_the_authored_prompt_comes_last(): + """It is closest to the conversation, and it is what the user actually + wrote.""" + joined = harness.join("HARNESS", "AUTHORED") + assert joined.index("HARNESS") < joined.index("AUTHORED") + + +def test_either_half_alone_is_returned_unchanged(): + assert harness.join("", "AUTHORED") == "AUTHORED" + assert harness.join("HARNESS", "") == "HARNESS" + assert harness.join("", "") == "" + + +# --- Through build_request --------------------------------------------------- +def _chat(db, owner, *, capabilities, model_prompt="", chat_prompt=""): + connection = Connection(name="c", base_url="http://h", api_key_encrypted="") + db.add(connection) + db.commit() + db.add( + Model( + connection_id=connection.id, + model_id="m", + capabilities_json=capabilities, + system_prompt=model_prompt, + ) + ) + db.commit() + chat = Chat( + user_id=owner.id, model_id="m", connection_id=connection.id, system_prompt=chat_prompt + ) + db.add(chat) + db.commit() + return chat + + +def _system(body): + first = body["messages"][0] if body["messages"] else {} + return first.get("content", "") if first.get("role") == "system" else "" + + +def test_a_request_without_tools_has_no_harness_and_no_tools_key(db, owner): + chat = _chat(db, owner, capabilities={}) + body = chat_service.build_request(db, chat, tools=[], user=owner) + assert "tools" not in body + assert "How to work" not in _system(body) + + +def test_the_harness_precedes_the_authored_prompt(db, owner): + settings_store.update(db, {"enabled": True}, key=settings_store.SEARCH) + chat = _chat(db, owner, capabilities={"tools": True}, chat_prompt="Speak as Gandalf.") + offered = tools_service.enabled_tools(db, chat, owner) + + system = _system(chat_service.build_request(db, chat, tools=offered, user=owner)) + assert system.index("How to work") < system.index("Speak as Gandalf.") + + +def test_precedence_between_the_authored_layers_is_untouched(db, owner): + """The harness is a different axis. Exactly one authored layer still wins, + and it is still the most specific one.""" + settings_store.update(db, {"system_prompt": "Instance."}) + settings_store.update(db, {"enabled": True}, key=settings_store.SEARCH) + + chat = _chat( + db, owner, capabilities={"tools": True}, model_prompt="Model.", chat_prompt="Chat." + ) + offered = tools_service.enabled_tools(db, chat, owner) + system = _system(chat_service.build_request(db, chat, tools=offered, user=owner)) + + assert "Chat." in system + assert "Model." not in system + assert "Instance." not in system + # And the resolver on its own is unchanged. + assert chat_service.effective_system_prompt(db, chat) == "Chat." + + +def test_a_model_prompt_wins_when_the_chat_has_none(db, owner): + settings_store.update(db, {"system_prompt": "Instance."}) + chat = _chat(db, owner, capabilities={}, model_prompt="Model.") + system = _system(chat_service.build_request(db, chat, user=owner)) + assert system == "Model." + + +def test_the_tools_array_rides_along(db, owner): + settings_store.update(db, {"enabled": True}, key=settings_store.SEARCH) + chat = _chat(db, owner, capabilities={"tools": True}) + offered = tools_service.enabled_tools(db, chat, owner) + body = chat_service.build_request(db, chat, tools=offered, user=owner) + assert body["tools"] == offered diff --git a/tests/test_library.py b/tests/test_library.py new file mode 100644 index 0000000..2d581a5 --- /dev/null +++ b/tests/test_library.py @@ -0,0 +1,291 @@ +"""The four stores: ingestion, search, memory limits, skill history.""" + +from __future__ import annotations + +import pytest +from sqlalchemy import text + +from lembas.db.migrations import ensure_fts +from lembas.db.models import AUTHOR_MODEL, Attachment, Document, User +from lembas.db.session import get_engine +from lembas.security.passwords import hash_password +from lembas.services import files as files_service +from lembas.services.library import documents as documents_service +from lembas.services.library import memories as memories_service +from lembas.services.library import notes as notes_service +from lembas.services.library import skills as skills_service +from lembas.services.library.fts import fts_query + + +@pytest.fixture +def owner(db): + user = User(name="Frodo", email="f@shire.test", password_hash=hash_password("x")) + db.add(user) + db.commit() + return user + + +# --- The index --------------------------------------------------------------- +def test_ensure_fts_is_idempotent(): + """It runs at every startup, like the column sync beside it.""" + assert ensure_fts(get_engine()) == [] + assert ensure_fts(get_engine()) == [] + + +def test_the_query_builder_survives_punctuation(): + """FTS5 MATCH has operators and a quoting rule, so a bare quote or asterisk + is a syntax error rather than a search that finds nothing.""" + assert fts_query('what "is" a mallorn?') == '"what" AND "is" AND "a" AND "mallorn"' + assert fts_query("a AND b OR NEAR *") == '"a" AND "AND" AND "b" AND "OR" AND "NEAR"' + assert fts_query(" ") == "" + assert fts_query("!!!") == "" + + +def test_editing_moves_a_note_in_the_index(db, owner): + """The triggers are what keep an external-content index correct.""" + note = notes_service.create(db, owner=owner, title="Tree", body="A golden tree.") + assert notes_service.search(db, owner, "golden") + + notes_service.update(db, note, body="A silver tree.") + assert notes_service.search(db, owner, "golden") == [] + assert notes_service.search(db, owner, "silver") + + +def test_a_deleted_note_leaves_the_index(db, owner): + note = notes_service.create(db, owner=owner, title="Tree", body="A golden tree.") + notes_service.delete(db, note) + assert notes_service.search(db, owner, "golden") == [] + + +def test_the_description_is_searched_as_well_as_the_contents(db, owner): + """A line of description is how somebody makes a document findable when its + own words do not include the term.""" + document = documents_service.store_upload( + db, owner=owner, payload=b"Opaque contents.", filename="a.txt", title="A" + ) + document.description = "Everything about invoicing." + db.commit() + assert documents_service.search(db, owner, "invoicing") + + +def test_a_natural_language_question_still_finds_the_document(db, owner): + """The caller is usually a model, which asks "who built the west gate of + Moria and what is its password" rather than "moria gate". Requiring every + term would lose the match on one absent word.""" + documents_service.store_upload( + db, + owner=owner, + payload=b"The west gate of Moria was built by Narvi. The password is mellon.", + filename="gate.txt", + title="Moria gate", + ) + found = documents_service.search( + db, owner, "who built the west gate of Moria and what is its password" + ) + assert [d.title for d in found] == ["Moria gate"] + + +def test_requiring_every_term_still_wins_when_it_can(db, owner): + """AND first, so a document matching all the words beats one matching some.""" + documents_service.store_upload( + db, owner=owner, payload=b"Golden mallorn trees of Lothlorien.", filename="a.txt", + title="Both", + ) + documents_service.store_upload( + db, owner=owner, payload=b"Golden light on the water.", filename="b.txt", title="One", + ) + found = documents_service.search(db, owner, "golden mallorn") + assert [d.title for d in found] == ["Both"] + + +def test_a_broken_index_does_not_break_the_page(db, owner): + """Search degrading to "finds nothing" is bad; a 500 on the library page is + worse, and a failed statement otherwise poisons the whole session.""" + notes_service.create(db, owner=owner, title="Tree", body="Golden.") + db.execute(text("DROP TABLE notes_fts")) + db.commit() + + assert notes_service.search(db, owner, "golden") == [] + # The session must still be usable afterwards. + assert notes_service.recent(db, owner) + + +# --- Ingestion --------------------------------------------------------------- +def test_a_document_and_an_attachment_extract_identically(db, owner): + """Both go through files.prepare, which is what guarantees the same file + produces the same text whichever way it arrived.""" + payload = b"# Waybread\n\nOne bite is enough." + + document = documents_service.store_upload( + db, owner=owner, payload=payload, filename="lembas.md" + ) + attachment = files_service.store( + db, user_id=owner.id, chat_id=None, payload=payload, filename="lembas.md" + ) + assert document.extracted_text == attachment.extracted_text + assert document.kind == attachment.kind + + +def test_a_document_keeps_its_own_file(db, owner): + document = documents_service.store_upload( + db, owner=owner, payload=b"hello", filename="a.txt" + ) + path = documents_service.stored_path(document.stored_name) + assert path is not None and path.read_bytes() == b"hello" + + +def test_the_library_path_check_refuses_an_escape(db): + """Same resolve-and-check as chat attachments, against a different root.""" + assert documents_service.stored_path("../../etc/passwd") is None + assert documents_service.stored_path("") is None + assert documents_service.stored_path(".hidden") is None + + +def test_attaching_a_document_copies_it(db, owner): + """History must not change under a conversation because a document was + edited or deleted later.""" + document = documents_service.store_upload( + db, owner=owner, payload=b"The ring is round.", filename="ring.txt" + ) + attachment = files_service.copy_document( + db, user_id=owner.id, chat_id=None, document=document + ) + documents_service.delete(db, document) + + db.refresh(attachment) + assert attachment.extracted_text == "The ring is round." + assert files_service.stored_path(attachment.stored_name) is not None + assert db.get(Document, document.id) is None + assert db.get(Attachment, attachment.id) is not None + + +# --- Memory ------------------------------------------------------------------ +def test_a_memory_is_trimmed_rather_than_refused(db, owner): + """A tool that writes an essay is told so and can put the long version in a + note; failing the write would just lose it.""" + memory = memories_service.add(db, owner=owner, content="x" * 5000) + assert len(memory.content) == memories_service.MAX_MEMORY_CHARS + + +def test_an_empty_memory_is_refused(db, owner): + with pytest.raises(ValueError): + memories_service.add(db, owner=owner, content=" ") + + +def test_memory_whitespace_is_collapsed(db, owner): + memory = memories_service.add(db, owner=owner, content=" two\n\nlines ") + assert memory.content == "two lines" + + +def test_the_injected_block_is_bounded(db, owner): + """Every memory costs tokens on every request, so the block has a ceiling. + Nothing disappears -- the full list is still in settings.""" + for index in range(200): + memories_service.add(db, owner=owner, content=f"Fact number {index}. " + "x" * 200) + + block = memories_service.block(db, owner) + assert len(block) < memories_service.MAX_TOTAL_CHARS + 200 + assert "more, see your settings" in block + + +def test_the_oldest_memories_survive_truncation(db, owner): + """A fact that has lasted is likelier to be a standing preference than + something said once this morning.""" + memories_service.add(db, owner=owner, content="The oldest fact.") + for index in range(100): + memories_service.add(db, owner=owner, content=f"Later fact {index}. " + "y" * 200) + + assert "The oldest fact." in memories_service.block(db, owner) + + +def test_there_is_a_hard_ceiling_on_records(db, owner, monkeypatch): + monkeypatch.setattr(memories_service, "MAX_RECORDS", 3) + for index in range(3): + memories_service.add(db, owner=owner, content=f"Fact {index}") + with pytest.raises(ValueError, match="note instead"): + memories_service.add(db, owner=owner, content="One too many") + + +def test_memory_block_of_nobody_is_empty(db): + assert memories_service.block(db, None) == "" + + +# --- Skills ------------------------------------------------------------------ +def test_a_skill_name_is_slugified(db, owner): + skill = skills_service.create( + db, owner=owner, name="Weekly Report!", description="When asked.", body="x" + ) + assert skill.name == "weekly-report" + + +def test_a_skill_needs_a_description(db, owner): + """It is the only thing the model sees until it opens the skill.""" + with pytest.raises(skills_service.SkillError, match="when to use it"): + skills_service.create(db, owner=owner, name="thing", description=" ", body="x") + + +def test_a_duplicate_name_is_refused(db, owner): + skills_service.create(db, owner=owner, name="report", description="When.", body="x") + with pytest.raises(skills_service.SkillError, match="already exists"): + skills_service.create(db, owner=owner, name="report", description="When.", body="y") + + +def test_every_edit_keeps_what_was_there(db, owner): + """The whole safety story for a model rewriting its own instructions.""" + skill = skills_service.create( + db, owner=owner, name="report", description="When.", body="First." + ) + skills_service.update(db, skill, body="Second.", author=AUTHOR_MODEL) + db.refresh(skill) + + assert skill.body == "Second." + assert skill.author == AUTHOR_MODEL + assert [r.body for r in skill.revisions] == ["First."] + + +def test_an_edit_that_changes_nothing_writes_no_revision(db, owner): + skill = skills_service.create( + db, owner=owner, name="report", description="When.", body="First." + ) + skills_service.update(db, skill, body="First.", description="When.") + db.refresh(skill) + assert skill.revisions == [] + + +def test_reverting_restores_and_is_itself_undoable(db, owner): + skill = skills_service.create( + db, owner=owner, name="report", description="When.", body="First." + ) + skills_service.update(db, skill, body="Second.", author=AUTHOR_MODEL) + db.refresh(skill) + + skills_service.revert(db, skill, skill.revisions[-1], author="user") + db.refresh(skill) + assert skill.body == "First." + # Going back is undoable too: the state before the revert was kept. + assert "Second." in [r.body for r in skill.revisions] + + +def test_the_index_lists_only_enabled_skills(db, owner): + skills_service.create(db, owner=owner, name="on", description="Use me.", body="x") + off = skills_service.create(db, owner=owner, name="off", description="Not me.", body="x") + skills_service.update(db, off, enabled=False) + + index = skills_service.index_block(db, owner) + assert "on: Use me." in index + assert "off" not in index + + +def test_the_index_is_name_and_description_only(db, owner): + """Bodies are fetched with skill_get; injecting them is what makes a hundred + skills unaffordable.""" + skills_service.create( + db, owner=owner, name="report", description="When asked.", body="SECRET STEPS" + ) + assert "SECRET STEPS" not in skills_service.index_block(db, owner) + + +def test_a_skill_is_found_by_the_name_a_model_would_use(db, owner): + skills_service.create(db, owner=owner, name="weekly-report", description="W.", body="x") + assert skills_service.by_name(db, "Weekly Report", owner) is not None + assert skills_service.by_name(db, "nope", owner) is None diff --git a/tests/test_sharing.py b/tests/test_sharing.py new file mode 100644 index 0000000..1ea6ffd --- /dev/null +++ b/tests/test_sharing.py @@ -0,0 +1,231 @@ +"""Who can see a document, a note or a skill. + +The most consequential tests in the library: everything else is a feature not +working, this is somebody reading somebody else's material. +""" + +from __future__ import annotations + +import pytest +from sqlalchemy import select + +from lembas.db.models import ( + PRINCIPAL_GROUP, + PRINCIPAL_USER, + Document, + Group, + Note, + Share, + User, +) +from lembas.security.passwords import hash_password +from lembas.services import sharing +from lembas.services.library import documents as documents_service +from lembas.services.library import notes as notes_service + + +@pytest.fixture +def people(db): + """Three accounts: an owner, a stranger, and an administrator.""" + made = {} + for name, role in (("frodo", "user"), ("gollum", "user"), ("gandalf", "admin")): + user = User( + name=name, email=f"{name}@shire.test", password_hash=hash_password("x"), role=role + ) + db.add(user) + made[name] = user + db.commit() + return made + + +def _note(db, owner, title="Secret"): + return notes_service.create(db, owner=owner, title=title, body="The ring is in the drawer.") + + +# --- The rule ---------------------------------------------------------------- +def test_the_owner_sees_their_own(db, people): + note = _note(db, people["frodo"]) + assert sharing.can_read(db, note, people["frodo"]) + assert note in db.scalars(notes_service.visible(db, people["frodo"])) + + +def test_a_stranger_sees_nothing(db, people): + note = _note(db, people["frodo"]) + assert not sharing.can_read(db, note, people["gollum"]) + assert note not in db.scalars(notes_service.visible(db, people["gollum"])) + + +def test_an_administrator_gets_no_free_pass(db, people): + """Admins bypass permissions elsewhere, deliberately -- an admin can grant + themselves those in two clicks. This is different: nobody made this + available to anyone, and administering a box is not being invited.""" + note = _note(db, people["frodo"]) + assert not sharing.can_read(db, note, people["gandalf"]) + assert note not in db.scalars(notes_service.visible(db, people["gandalf"])) + + +def test_sharing_with_a_person_lets_them_read(db, people): + note = _note(db, people["frodo"]) + sharing.set_grants(db, note, user_ids=[people["gollum"].id], group_ids=[]) + assert sharing.can_read(db, note, people["gollum"]) + assert note in db.scalars(notes_service.visible(db, people["gollum"])) + + +def test_sharing_with_a_group_lets_its_members_read(db, people): + group = Group(name="Fellowship") + group.users.append(people["gollum"]) + db.add(group) + db.commit() + + note = _note(db, people["frodo"]) + sharing.set_grants(db, note, user_ids=[], group_ids=[group.id]) + assert sharing.can_read(db, note, people["gollum"]) + + +def test_leaving_a_group_takes_the_access_with_it(db, people): + group = Group(name="Fellowship") + group.users.append(people["gollum"]) + db.add(group) + db.commit() + note = _note(db, people["frodo"]) + sharing.set_grants(db, note, user_ids=[], group_ids=[group.id]) + + group.users.remove(people["gollum"]) + db.commit() + db.refresh(people["gollum"]) + assert not sharing.can_read(db, note, people["gollum"]) + + +def test_signed_out_sees_nothing(db, people): + _note(db, people["frodo"]) + assert list(db.scalars(notes_service.visible(db, None))) == [] + + +# --- Sharing grants reading only --------------------------------------------- +def test_a_share_does_not_grant_writing(db, people): + """Two people editing one note with no history and no merge is worse than + the inconvenience of copying it.""" + note = _note(db, people["frodo"]) + sharing.set_grants(db, note, user_ids=[people["gollum"].id], group_ids=[]) + assert sharing.can_read(db, note, people["gollum"]) + assert not sharing.can_write(note, people["gollum"]) + assert sharing.can_write(note, people["frodo"]) + + +# --- Managing grants --------------------------------------------------------- +def test_set_grants_replaces_rather_than_adds(db, people): + note = _note(db, people["frodo"]) + sharing.set_grants(db, note, user_ids=[people["gollum"].id], group_ids=[]) + sharing.set_grants(db, note, user_ids=[people["gandalf"].id], group_ids=[]) + + assert not sharing.can_read(db, note, people["gollum"]) + assert sharing.can_read(db, note, people["gandalf"]) + + +def test_sharing_with_yourself_is_ignored(db, people): + note = _note(db, people["frodo"]) + sharing.set_grants(db, note, user_ids=[people["frodo"].id], group_ids=[]) + assert sharing.grants_for(db, note) == [] + + +def test_deleting_a_note_drops_its_shares(db, people): + """Shares carry no foreign key to their resource -- one column pointing at + three tables cannot have one -- so nothing cascades.""" + note = _note(db, people["frodo"]) + sharing.set_grants(db, note, user_ids=[people["gollum"].id], group_ids=[]) + notes_service.delete(db, note) + assert db.scalar(select(Share).where(Share.resource_id == note.id)) is None + + +def test_forgetting_a_principal_drops_their_shares(db, people): + """A stale row would grant access to whoever next received that id.""" + note = _note(db, people["frodo"]) + sharing.set_grants(db, note, user_ids=[people["gollum"].id], group_ids=[]) + assert sharing.forget_principal(db, PRINCIPAL_USER, people["gollum"].id) == 1 + assert sharing.grants_for(db, note) == [] + + +def test_two_kinds_of_resource_do_not_collide(db, people): + """One shares table across three resource types, so the type must be part + of the match -- otherwise a note and a document sharing an id would share + each other's access.""" + note = _note(db, people["frodo"]) + document = documents_service.store_upload( + db, owner=people["frodo"], payload=b"hello", filename="a.txt" + ) + sharing.set_grants(db, note, user_ids=[people["gollum"].id], group_ids=[]) + + assert sharing.can_read(db, note, people["gollum"]) + assert not sharing.can_read(db, document, people["gollum"]) + + +def test_resource_type_refuses_something_unshareable(db, people): + """Memory is deliberately not shareable: a record about a person is not + content to hand round.""" + from lembas.db.models import Memory + + with pytest.raises(ValueError): + sharing.resource_type(Memory) + + +# --- Through the search path ------------------------------------------------- +def test_search_does_not_leak_across_owners(db, people): + """The index is searched first and the visibility filter applied to what it + returned. Getting that order wrong leaks a hit even without the contents.""" + notes_service.create( + db, owner=people["frodo"], title="Mallorn", body="A golden tree of Lothlorien." + ) + assert notes_service.search(db, people["frodo"], "golden") + assert notes_service.search(db, people["gollum"], "golden") == [] + assert notes_service.search(db, people["gandalf"], "golden") == [] + + +def test_a_shared_document_is_findable_by_the_person_it_was_shared_with(db, people): + document = documents_service.store_upload( + db, + owner=people["frodo"], + payload=b"The mallorn is a golden tree.", + filename="tree.txt", + ) + assert documents_service.search(db, people["gollum"], "mallorn") == [] + + sharing.set_grants(db, document, user_ids=[people["gollum"].id], group_ids=[]) + found = documents_service.search(db, people["gollum"], "mallorn") + assert [d.id for d in found] == [document.id] + + +def test_the_shares_table_records_what_was_asked_for(db, people): + group = Group(name="Fellowship") + db.add(group) + db.commit() + note = _note(db, people["frodo"]) + sharing.set_grants( + db, note, user_ids=[people["gollum"].id], group_ids=[group.id] + ) + kinds = {(s.principal_type, s.principal_id) for s in sharing.grants_for(db, note)} + assert kinds == { + (PRINCIPAL_USER, people["gollum"].id), + (PRINCIPAL_GROUP, group.id), + } + + +def test_visibility_is_a_query_filter_not_a_python_loop(db, people): + """visible_to returns a condition so callers can page and order on the + database side; a Python filter would break pagination silently.""" + for index in range(3): + _note(db, people["frodo"], title=f"Note {index}") + _note(db, people["gollum"], title="Theirs") + + rows = db.scalars( + notes_service.visible(db, people["frodo"]).order_by(Note.title).limit(2) + ) + assert [n.title for n in rows] == ["Note 0", "Note 1"] + + +def test_documents_and_notes_use_the_same_rule(db, people): + document = documents_service.store_upload( + db, owner=people["frodo"], payload=b"x", filename="a.txt" + ) + assert document in db.scalars(documents_service.visible(db, people["frodo"])) + assert document not in db.scalars(documents_service.visible(db, people["gollum"])) + assert list(db.scalars(select(Document).where(sharing.visible_to(Document, None)))) == [] diff --git a/tests/test_tools.py b/tests/test_tools.py index 8c75c68..2dc1905 100644 --- a/tests/test_tools.py +++ b/tests/test_tools.py @@ -104,12 +104,19 @@ def _user(db, user_id): return db.get(User, user_id) -def test_nothing_is_offered_when_search_is_off(db, user_id): +def _names(offered): + return {tool["function"]["name"] for tool in offered} + + +def test_web_search_is_absent_when_search_is_off(db, user_id): + """The library tools do not depend on a search provider, so they stay.""" chat = _chat_with(db, user_id, capabilities={"tools": True}) - assert tools_service.enabled_tools(db, chat, _user(db, user_id)) == [] + offered = _names(tools_service.enabled_tools(db, chat, _user(db, user_id))) + assert "web_search" not in offered + assert "notes_search" in offered -def test_nothing_is_offered_to_a_model_without_the_tools_capability(db, user_id): +def test_nothing_at_all_without_the_tools_capability(db, user_id): """Sending a tools array to an endpoint that does not implement tool calling fails the entire request, exactly as image parts do without vision.""" settings_store.update(db, {"enabled": True}, key=settings_store.SEARCH) @@ -120,13 +127,29 @@ def test_nothing_is_offered_to_a_model_without_the_tools_capability(db, user_id) def test_web_search_is_offered_when_everything_lines_up(db, user_id): settings_store.update(db, {"enabled": True}, key=settings_store.SEARCH) chat = _chat_with(db, user_id, capabilities={"tools": True}) - - offered = tools_service.enabled_tools(db, chat, _user(db, user_id)) - assert len(offered) == 1 - assert offered[0]["function"]["name"] == "web_search" + assert "web_search" in _names(tools_service.enabled_tools(db, chat, _user(db, user_id))) -def test_nothing_is_offered_when_the_provider_cannot_run(db, user_id, monkeypatch): +def test_a_model_predating_the_split_keeps_its_tools(db, user_id): + """Rows configured before the per-tool flags existed have no tool_* keys. + Reading absent as off would silently take web search away from every model + already set up for it.""" + settings_store.update(db, {"enabled": True}, key=settings_store.SEARCH) + chat = _chat_with(db, user_id, capabilities={"tools": True}) + assert "web_search" in _names(tools_service.enabled_tools(db, chat, _user(db, user_id))) + + +def test_a_family_turned_off_for_the_model_is_withheld(db, user_id): + settings_store.update(db, {"enabled": True}, key=settings_store.SEARCH) + chat = _chat_with( + db, user_id, capabilities={"tools": True, "tool_notes": False} + ) + offered = _names(tools_service.enabled_tools(db, chat, _user(db, user_id))) + assert "notes_search" not in offered + assert "web_search" in offered, "turning one family off must not affect another" + + +def test_web_search_is_withheld_when_the_provider_cannot_run(db, user_id, monkeypatch): """Offering a tool that will fail on every call is worse than not offering it at all.""" monkeypatch.setattr( @@ -134,7 +157,13 @@ def test_nothing_is_offered_when_the_provider_cannot_run(db, user_id, monkeypatc ) settings_store.update(db, {"enabled": True}, key=settings_store.SEARCH) chat = _chat_with(db, user_id, capabilities={"tools": True}) - assert tools_service.enabled_tools(db, chat, _user(db, user_id)) == [] + assert "web_search" not in _names( + tools_service.enabled_tools(db, chat, _user(db, user_id)) + ) + + +def _context(**kwargs): + return tools_service.ToolContext(owner_id="someone", **kwargs) # --- Running one ------------------------------------------------------------- @@ -144,7 +173,7 @@ async def test_running_web_search_formats_results_for_the_model(monkeypatch): monkeypatch.setattr("lembas.services.search.run", fake_run) - outcome = await tools_service.run_tool({}, "web_search", '{"query": "mallorn"}') + outcome = await tools_service.run_tool(_context(), "web_search", '{"query": "mallorn"}') assert "A title" in outcome.content assert "https://a.test" in outcome.content assert outcome.event["status"] == "ok" @@ -161,7 +190,7 @@ async def test_malformed_argument_json_is_treated_as_the_query(monkeypatch): return [] monkeypatch.setattr("lembas.services.search.run", fake_run) - await tools_service.run_tool({}, "web_search", "mallorn tree") + await tools_service.run_tool(_context(), "web_search", "mallorn tree") assert seen["query"] == "mallorn tree" @@ -174,19 +203,19 @@ async def test_a_failed_search_hands_the_model_an_explanation(monkeypatch): monkeypatch.setattr("lembas.services.search.run", fake_run) - outcome = await tools_service.run_tool({}, "web_search", '{"query": "x"}') + outcome = await tools_service.run_tool(_context(), "web_search", '{"query": "x"}') assert "rate limiting" in outcome.content assert outcome.event["status"] == "error" async def test_an_unknown_tool_is_reported_rather_than_raised(): - outcome = await tools_service.run_tool({}, "launch_missiles", "{}") + outcome = await tools_service.run_tool(_context(), "launch_missiles", "{}") assert "no tool called" in outcome.content assert outcome.event["status"] == "error" async def test_a_call_with_no_query_is_reported(): - outcome = await tools_service.run_tool({}, "web_search", '{"query": " "}') + outcome = await tools_service.run_tool(_context(), "web_search", '{"query": " "}') assert outcome.event["status"] == "error" @@ -220,8 +249,16 @@ def test_the_tool_turn_carries_the_call_id(): } -def test_the_schema_is_valid_json(): - """It is sent verbatim to the endpoint; a schema that will not serialise +def test_every_schema_is_valid_json(): + """They are sent verbatim to the endpoint; a schema that will not serialise fails every request rather than one.""" - json.dumps(tools_service.WEB_SEARCH_SCHEMA) - assert tools_service.WEB_SEARCH_SCHEMA["function"]["parameters"]["required"] == ["query"] + for name, tool in tools_service.REGISTRY.items(): + json.dumps(tool.schema) + assert tool.schema["function"]["name"] == name + assert tool.family in tools_service.FAMILIES + + +def test_every_tool_describes_when_to_use_it(): + """The description is all the model has to decide with.""" + for tool in tools_service.REGISTRY.values(): + assert len(tool.description) > 40, tool.name