Houmeres/LLeMbas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Knowledge bases, and a file input that lines up

Browse Source 
**Bases.** Documents now live in named collections rather than one flat pile,
and a chat can be pointed at particular ones — "answer from the contracts
folder" is a different question from "answer from everything I have ever
uploaded". A chat with none attached still searches everything its owner can
see, because empty means unscoped, not empty.

The harness names the attached bases. Without that the model cannot tell "there
is nothing about this" from "I am only allowed to see one folder", and it
phrases a miss as the former.

**Sharing moves to the base.** A document is visible to whoever can see the base
it lives in, so `Document` is gone from the shareable types and
`documents.visible()` filters through `base_id`. "This folder is the team's" is
the granularity people think in; per-document grants meant answering "who can
see this?" by checking every file. Moving a document between bases changes who
can see it, so the destination has to be one you own.

`Document.base_id` is nullable only because the column had to be added to a
table that already had rows. `sweep_unfiled()` runs at startup beside the
orphaned-upload sweep and files anything predating bases into its owner's
default, which is what makes "always set" true everywhere else.

**The file input.** `.input` gave it a fixed height and horizontal padding, so
the browser's own button sat hard against the left edge while the filename
floated off the centre line. A file input is two controls in one box and
neither inherits anything useful, so it gets its own rule: no horizontal
padding, the button sized to `--control-h` with the divider that separates it,
and the text centred with line-height rather than flexbox, which file inputs do
not lay out reliably.

437 tests, ruff clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
2026-07-21 20:00:15 +02:00
parent 21001f2eb8
commit a0f733063a
24 changed files with 836 additions and 143 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lembas/services/library/documents.py
+138 -10
View File
@@ -18,7 +18,7 @@ from sqlalchemy import select
from sqlalchemy.orm import Session as DBSession
from lembas.config import settings
from lembas.db.models import SOURCE_LINK, SOURCE_UPLOAD, Document, User
from lembas.db.models import SOURCE_LINK, SOURCE_UPLOAD, Document, KnowledgeBase, User
from lembas.services import files as files_service
from lembas.services import sharing
from lembas.services.fetch import Fetched
@@ -28,6 +28,10 @@ log = logging.getLogger(__name__)
INDEX = "documents_fts"
# What a first base is called when one has to be invented -- on the first
# upload, or for documents that predate bases existing.
DEFAULT_BASE_NAME = "My documents"
# How much of a document's text a search result carries back to the model. A
# whole 100-page extract would swallow the context window; this is enough to
# judge relevance and to answer from, and `knowledge_get` fetches the rest.
@@ -57,12 +61,109 @@ def stored_path(stored_name: str) -> Path | None:
return path if path.is_file() else None
# --- Bases -------------------------------------------------------------------
def visible_bases(db: DBSession, user: User | None):
return select(KnowledgeBase).where(sharing.visible_to(KnowledgeBase, user))
def get_base(db: DBSession, base_id: str, user: User | None) -> KnowledgeBase | None:
base = db.get(KnowledgeBase, base_id)
if base is None or not sharing.can_read(db, base, user):
return None
return base
def create_base(
db: DBSession, *, owner: User, name: str, description: str = ""
) -> KnowledgeBase:
name = " ".join((name or "").split())[:200] or DEFAULT_BASE_NAME
existing = db.scalar(
select(KnowledgeBase).where(
KnowledgeBase.owner_id == owner.id, KnowledgeBase.name == name
)
)
if existing is not None:
raise ValueError(f"You already have a knowledge base called {name!r}.")
base = KnowledgeBase(owner_id=owner.id, name=name, description=description.strip()[:2000])
db.add(base)
db.commit()
return base
def default_base(db: DBSession, owner: User) -> KnowledgeBase:
"""The base a document goes into when none was chosen.
Made on demand rather than at registration, so an account that never uses
the library never grows an empty one.
"""
base = db.scalar(
select(KnowledgeBase)
.where(KnowledgeBase.owner_id == owner.id)
.order_by(KnowledgeBase.created_at)
)
if base is not None:
return base
base = KnowledgeBase(owner_id=owner.id, name=DEFAULT_BASE_NAME)
db.add(base)
db.commit()
return base
def delete_base(db: DBSession, base: KnowledgeBase) -> None:
"""Delete a base and everything in it.
The documents go too -- a base is a place, not a label, and leaving its
contents behind with nowhere to live would need an "unfiled" concept that
exists only to hold the wreckage of deletes.
"""
for document in list(base.documents):
path = stored_path(document.stored_name)
if path is not None:
path.unlink(missing_ok=True)
sharing.forget_resource(db, base)
db.delete(base)
db.commit()
def sweep_unfiled(db: DBSession) -> int:
"""File documents that predate knowledge bases into their owner's default.
`Document.base_id` is nullable only because the column had to be added to a
table that already had rows. This is what makes "always set" true in
practice, and it runs at startup beside the orphaned-upload sweep.
"""
unfiled = list(db.scalars(select(Document).where(Document.base_id.is_(None))))
if not unfiled:
return 0
bases: dict[str, KnowledgeBase] = {}
for document in unfiled:
owner = db.get(User, document.owner_id)
if owner is None:
continue
if owner.id not in bases:
bases[owner.id] = default_base(db, owner)
document.base_id = bases[owner.id].id
db.commit()
log.info("filed %d document(s) that predated knowledge bases", len(unfiled))
return len(unfiled)
# --- Creating ----------------------------------------------------------------
def store_upload(
db: DBSession, *, owner: User, payload: bytes, filename: str, title: str = ""
db: DBSession,
*,
owner: User,
payload: bytes,
filename: str,
title: str = "",
base: KnowledgeBase | None = None,
) -> Document:
"""Add an uploaded file to the library. Raises files.FileError if unusable."""
prepared = files_service.prepare(payload, filename)
base = base or default_base(db, owner)
stored_name = f"{secrets.token_hex(16)}{prepared.extension}"
(library_dir() / stored_name).write_bytes(prepared.payload)
@@ -70,6 +171,7 @@ def store_upload(
display = files_service.safe_display_name(filename)
document = Document(
owner_id=owner.id,
base_id=base.id,
title=(title.strip() or display)[:300],
source=SOURCE_UPLOAD,
filename=display,
@@ -90,14 +192,18 @@ def store_upload(
return document
def store_page(db: DBSession, *, owner: User, page: Fetched) -> Document:
def store_page(
db: DBSession, *, owner: User, page: Fetched, base: KnowledgeBase | None = None
) -> Document:
"""Add a fetched web page to the library.
Saved as text rather than as the original HTML: the point of keeping it is
what it said, and the markup would have to be reduced again on every read.
"""
base = base or default_base(db, owner)
document = Document(
owner_id=owner.id,
base_id=base.id,
title=page.title[:300] or page.url[:300],
source=SOURCE_LINK,
source_url=page.url,
@@ -115,19 +221,41 @@ def store_page(db: DBSession, *, owner: User, page: Fetched) -> Document:
# --- Reading -----------------------------------------------------------------
def visible(db: DBSession, user: User | None):
return select(Document).where(sharing.visible_to(Document, user))
def visible(db: DBSession, user: User | None, *, base_ids: list[str] | None = None):
"""Documents this user may see, optionally narrowed to some bases.
Visibility comes from the base, not the document: a document is readable by
whoever can read the base it lives in. That is the whole reason bases are
shareable and documents are not.
"""
condition = Document.base_id.in_(
select(KnowledgeBase.id).where(sharing.visible_to(KnowledgeBase, user))
)
query = select(Document).where(condition)
if base_ids:
# Still filtered by visibility above, so naming a base you cannot see
# returns nothing rather than granting access to it.
query = query.where(Document.base_id.in_(base_ids))
return query
def get(db: DBSession, document_id: str, user: User | None) -> Document | None:
document = db.get(Document, document_id)
if document is None or not sharing.can_read(db, document, user):
if document is None:
return None
base = db.get(KnowledgeBase, document.base_id) if document.base_id else None
if base is None or not sharing.can_read(db, base, user):
return None
return document
def search(
db: DBSession, user: User | None, needle: str, *, limit: int = 10
db: DBSession,
user: User | None,
needle: str,
*,
limit: int = 10,
base_ids: list[str] | None = None,
) -> list[Document]:
"""Documents matching `needle` that this user may see, best match first.
@@ -141,7 +269,9 @@ def search(
order = {hit.id: position for position, hit in enumerate(hits)}
rows = list(
db.scalars(visible(db, user).where(Document.id.in_(list(order))))
db.scalars(
visible(db, user, base_ids=base_ids).where(Document.id.in_(list(order)))
)
)
rows.sort(key=lambda document: order.get(document.id, len(order)))
return rows[:limit]
@@ -160,7 +290,5 @@ def delete(db: DBSession, document: Document) -> None:
path = stored_path(document.stored_name)
if path is not None:
path.unlink(missing_ok=True)
# Shares carry no foreign key to their resource, so nothing cascades.
sharing.forget_resource(db, document)
db.delete(document)
db.commit()