Data groups: a provider's models read only their own group's data

Every connection is in a data group. Its models are handed, and can find,
only that group's memories, notes, skills, knowledge, reports and
personality -- by search and by id. A chat stays in the group it was
started in: switching its model, the endpoint fallback, the crowd, friends,
bases and the @ menu all stay inside it, and a chat whose model has moved
is refused rather than sent. A group may name its own embedder and image
reviewer. data.manage lets a person make personal groups, remap
connections for themselves and move their own records.

Also: a search no longer mixes two embedders of the same width.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-29 16:07:55 +00:00
co-authored by Claude Opus 5.5
parent e65ea90fe6
commit 9970bb43c6
61 changed files with 3595 additions and 200 deletions
+245
View File
@@ -0,0 +1,245 @@
"""A chat stays in the data group it was started in.
Its history is the group's data, so every way a chat could reach a model in
another group is closed here: switching its model, the endpoint fallback, a
crowd member, a friend, the roster, a base, the `@` menu, and Messages. And if
a chat's own model is moved into another group afterwards, the next reply is
refused rather than sent.
"""
from __future__ import annotations
import pytest
from sqlalchemy import select
from lembas.db.models import (
DEFAULT_GROUP,
TARGET_MESSAGES,
Chat,
Connection,
CrowdMember,
DataGroup,
Model,
User,
)
from lembas.services import chat as chat_service
from lembas.services import data_groups, schedules, settings_store
from lembas.services import subagent as subagent_service
from lembas.services.crypto import encrypt
from lembas.services.library import notes
HOSTED = "hosted"
@pytest.fixture
def owner(db, registered) -> User:
return db.scalars(select(User).order_by(User.created_at)).first()
@pytest.fixture
def setup(db, owner):
"""Two local models in the default group, one hosted model in its own."""
db.add(DataGroup(id=HOSTED, name="Hosted"))
local = Connection(name="Local", base_url="http://127.0.0.1:1", api_key_encrypted=encrypt(""))
cloud = Connection(
name="Cloud",
base_url="http://127.0.0.1:2",
api_key_encrypted=encrypt(""),
data_group_id=HOSTED,
)
db.add_all([local, cloud])
db.flush()
db.add_all(
[
Model(connection_id=local.id, model_id="local-a", position=0),
Model(connection_id=local.id, model_id="local-b", position=1),
Model(connection_id=cloud.id, model_id="cloud-model", position=2),
]
)
db.commit()
return local, cloud
def _chat(db, owner, model_id="local-a", connection=None, group=DEFAULT_GROUP) -> Chat:
chat = Chat(
user_id=owner.id,
model_id=model_id,
connection_id=connection.id if connection else None,
data_group_id=group,
title="t",
)
db.add(chat)
db.commit()
return chat
# --- Starting and switching ---------------------------------------------------------
def test_a_new_chat_takes_its_models_group(client, db, setup):
client.post("/api/chats/start", data={"content": "hi", "model_id": "cloud-model"})
chat = db.scalars(select(Chat).order_by(Chat.created_at.desc())).first()
assert chat.data_group_id == HOSTED
def test_a_chat_cannot_be_switched_to_another_groups_model(client, db, owner, setup):
local, _ = setup
chat = _chat(db, owner, connection=local)
response = client.patch(f"/api/chats/{chat.id}", data={"model_id": "cloud-model"})
assert response.status_code == 409
assert "data group" in response.text
db.refresh(chat)
assert chat.model_id == "local-a"
def test_a_chat_can_switch_within_its_group(client, db, owner, setup):
local, _ = setup
chat = _chat(db, owner, connection=local)
response = client.patch(f"/api/chats/{chat.id}", data={"model_id": "local-b"})
assert response.status_code in (200, 204)
db.refresh(chat)
assert chat.model_id == "local-b"
def test_the_picker_names_the_models_it_leaves_out(client, db, owner, setup):
"""Named, not silently missing -- and not offered as options either."""
local, _ = setup
chat = _chat(db, owner, connection=local)
page = client.get(f"/chat/{chat.id}").text
assert "In another data group" in page
assert 'data-picker-value="cloud-model"' not in page
def test_available_models_narrow_to_a_group(db, owner, setup):
ids = [m.model_id for m in chat_service.available_models(db, owner, HOSTED)]
assert ids == ["cloud-model"]
everything = [m.model_id for m in chat_service.available_models(db, owner)]
assert everything == ["local-a", "local-b", "cloud-model"]
# --- The endpoint fallback ----------------------------------------------------------
def test_the_fallback_never_repoints_a_chat_into_another_group(db, owner, setup):
"""A model id served by two connections: the chat's own going away must not
land it on the other provider, which would be handed the whole history."""
local, cloud = setup
db.add(Model(connection_id=cloud.id, model_id="local-a"))
local.enabled = False
db.commit()
chat = _chat(db, owner, connection=local)
with pytest.raises(chat_service.LLMError):
chat_service.resolve_endpoint(db, chat)
db.refresh(chat)
assert chat.connection_id == local.id
def test_a_chat_whose_model_moved_is_refused_rather_than_sent(db, owner, setup):
local, _ = setup
chat = _chat(db, owner, connection=local)
speaker = chat_service.speaker_for(db, chat)
assert data_groups.refusal(db, owner, chat, speaker) == ""
local.data_group_id = HOSTED
db.commit()
refusal = data_groups.refusal(db, owner, chat, speaker)
assert "Default" in refusal and "Hosted" in refusal
# --- Other models reaching the conversation ------------------------------------------
def test_a_crowd_member_from_another_group_is_refused(client, db, owner, setup):
settings_store.update(db, {"enabled": True}, key=settings_store.CROWD)
local, _ = setup
chat = _chat(db, owner, connection=local)
client.patch(f"/api/chats/{chat.id}", data={"crowd_model_ids": ["local-b", "cloud-model"]})
members = [row.model_id for row in db.scalars(select(CrowdMember))]
assert members == ["local-b"]
def test_a_crowd_member_that_left_the_group_does_not_speak(db, owner, setup):
local, cloud = setup
chat = _chat(db, owner, connection=local)
db.add(CrowdMember(chat_id=chat.id, model_id="cloud-model", connection_id=cloud.id))
db.commit()
from lembas.services import crowd
speakers = [s.model_id for s in crowd.member_speakers(db, chat, owner)]
assert speakers == ["local-a"]
def test_the_roster_and_the_friend_stay_in_the_group(db, owner, setup):
roster = chat_service.roster_block(db, owner, exclude="local-a", group=DEFAULT_GROUP)
assert "local-b" in roster and "cloud-model" not in roster
friend, refusal = subagent_service._resolve_friend(
db, owner, "cloud-model", asking="local-a", group=DEFAULT_GROUP
)
assert friend is None
# The name asked for is echoed back; the list of who *can* be asked is not
# allowed to carry it.
offered = refusal.split("These are the ones you can:")[-1]
assert "cloud-model" not in offered and "local-b" in offered
def test_a_friend_reads_its_own_group(db, owner, setup):
local, cloud = setup
parent = _chat(db, owner, connection=local)
friend = db.scalar(select(Model).where(Model.model_id == "cloud-model"))
child = subagent_service._create_child(db, parent, title="q", write=False, friend=friend)
assert child.data_group_id == HOSTED
# --- Bases and the @ menu -------------------------------------------------------------
def test_a_base_from_another_group_cannot_be_attached(client, db, owner, setup):
from lembas.services.library import documents
local, _ = setup
chat = _chat(db, owner, connection=local)
base = documents.create_base(db, owner=owner, name="Hosted base", group=HOSTED)
response = client.post(f"/api/chats/{chat.id}/bases", data={"base_id": base.id})
assert response.status_code == 404
def test_the_mention_menu_offers_only_the_chats_group(client, db, owner, setup):
local, _ = setup
chat = _chat(db, owner, connection=local)
notes.create(db, owner=owner, title="Home note", body="x")
notes.create(db, owner=owner, title="Hosted note", body="x", group=HOSTED)
page = client.get(f"/api/files/mention-picker?q=&chat_id={chat.id}").text
assert "Home note" in page and "Hosted note" not in page
def test_on_the_new_chat_screen_the_chosen_model_decides(client, db, owner, setup):
notes.create(db, owner=owner, title="Hosted note", body="x", group=HOSTED)
page = client.get("/api/files/mention-picker?q=&model_id=cloud-model").text
assert "Hosted note" in page
def test_a_note_from_another_group_cannot_be_attached(client, db, owner, setup):
local, _ = setup
chat = _chat(db, owner, connection=local)
hosted = notes.create(db, owner=owner, title="Hosted note", body="x", group=HOSTED)
response = client.post(
"/api/files/from-note", data={"note_id": hosted.id, "chat_id": chat.id}
)
assert "not available" in response.text
# --- Messages and schedules -------------------------------------------------------------
def test_a_schedule_from_another_group_cannot_post_to_messages(db, owner, setup):
rule = {"at": {"weekdays": [0], "times": ["15:00"]}}
with pytest.raises(schedules.ScheduleError, match="Messages"):
schedules.create(
db,
owner=owner,
title="t",
instruction="i",
rule=rule,
target=TARGET_MESSAGES,
model_id="cloud-model",
)
def test_a_schedule_is_stamped_with_its_models_group(db, owner, setup):
rule = {"at": {"weekdays": [0], "times": ["15:00"]}}
schedule = schedules.create(
db, owner=owner, title="t", instruction="i", rule=rule, model_id="cloud-model"
)
assert schedule.data_group_id == HOSTED
assert db.get(Chat, schedule.chat_id).data_group_id == HOSTED
+221
View File
@@ -0,0 +1,221 @@
"""A model reads one data group's data, and only that one -- from both sides.
Every store is checked twice: in the harness, where memories, skills and a
personality are *handed* to a model, and in the tool runners, where a model goes
looking. A test that only covered the search would miss the fetch by id, which is
the path a model takes after learning an id from somewhere it should not have.
"""
from __future__ import annotations
import json
import pytest
from sqlalchemy import select
from lembas.db.models import (
AUTHOR_MODEL,
DEFAULT_GROUP,
Chat,
Connection,
DataGroup,
Impression,
Model,
User,
)
from lembas.services import harness, personas, reports
from lembas.services import tools as tools_service
from lembas.services.crypto import encrypt
from lembas.services.library import documents, memories, notes, skills
HOSTED = "hosted"
TOOLS = {"tools": True}
@pytest.fixture
def owner(db, registered) -> User:
return db.scalars(select(User).order_by(User.created_at)).first()
@pytest.fixture
def chats(db, owner) -> tuple[Chat, Chat]:
"""One chat in the default group, one in the hosted group."""
db.add(DataGroup(id=HOSTED, name="Hosted"))
local = Connection(name="Local", base_url="http://127.0.0.1:1", api_key_encrypted=encrypt(""))
cloud = Connection(
name="Cloud",
base_url="http://127.0.0.1:2",
api_key_encrypted=encrypt(""),
data_group_id=HOSTED,
)
db.add_all([local, cloud])
db.flush()
db.add_all(
[
Model(connection_id=local.id, model_id="local-model", capabilities_json=TOOLS),
Model(connection_id=cloud.id, model_id="cloud-model", capabilities_json=TOOLS),
]
)
home = Chat(user_id=owner.id, model_id="local-model", connection_id=local.id,
data_group_id=DEFAULT_GROUP)
away = Chat(user_id=owner.id, model_id="cloud-model", connection_id=cloud.id,
data_group_id=HOSTED)
db.add_all([home, away])
db.commit()
return home, away
def _tools(*names):
return [tools_service.REGISTRY[name].schema for name in names]
def _context(db, owner, chat) -> tools_service.ToolContext:
return tools_service.context_for(db, owner, chat, tools=None)
async def _run(context, tool: str, **args):
return await tools_service.run_tool(context, tool, json.dumps(args))
# --- The harness: what a model is handed ------------------------------------------
def test_a_model_is_handed_only_its_own_groups_memories(db, owner, chats):
home, away = chats
memories.add(db, owner=owner, content="Home fact.", group=DEFAULT_GROUP)
memories.add(db, owner=owner, content="Hosted fact.", group=HOSTED)
at_home = harness.compose(db, owner, _tools("memory_add"), chat=home)
abroad = harness.compose(db, owner, _tools("memory_add"), chat=away)
assert "Home fact." in at_home and "Hosted fact." not in at_home
assert "Hosted fact." in abroad and "Home fact." not in abroad
def test_the_skill_index_is_one_groups(db, owner, chats):
home, away = chats
skills.create(db, owner=owner, name="home-skill", description="Home.", body="b")
skills.create(
db, owner=owner, name="away-skill", description="Away.", body="b", group=HOSTED
)
abroad = harness.compose(db, owner, _tools("skill_get"), chat=away)
assert "away-skill" in abroad
assert "home-skill" not in abroad
def test_a_personality_is_per_group(db, owner, chats):
home, away = chats
personas.write(
db,
model_key=personas.key_for("cloud-model", HOSTED),
owner=owner,
content="The hosted self.",
author=AUTHOR_MODEL,
)
abroad = harness.compose(db, owner, _tools("persona_write"), chat=away)
assert "The hosted self." in abroad
def test_a_base_in_another_group_is_not_named(db, owner, chats):
home, away = chats
base = documents.create_base(db, owner=owner, name="Home contracts")
away.knowledge_bases = [base]
db.commit()
abroad = harness.compose(db, owner, _tools("knowledge_search"), chat=away)
assert "Home contracts" not in abroad
# --- The tools: what a model can go and get ----------------------------------------
def test_the_tool_context_carries_the_chats_group(db, owner, chats):
home, away = chats
assert _context(db, owner, home).data_group == DEFAULT_GROUP
assert _context(db, owner, away).data_group == HOSTED
async def test_a_note_in_another_group_cannot_be_searched_or_fetched(db, owner, chats):
home, away = chats
secret = notes.create(db, owner=owner, title="Home only", body="mallorn", group=DEFAULT_GROUP)
context = _context(db, owner, away)
found = await _run(context, "notes_search", query="mallorn")
assert found.event["results"] == []
fetched = await _run(context, "notes_get", id=secret.id)
assert fetched.event["status"] == "error"
edited = await _run(context, "notes_edit", id=secret.id, body="gone")
assert edited.event["status"] == "error"
async def test_a_note_a_model_writes_lands_in_its_group(db, owner, chats):
home, away = chats
outcome = await _run(_context(db, owner, away), "notes_create", title="t", body="b")
assert outcome.event["status"] == "ok"
note = notes.get(db, outcome.event["results"][0]["id"], owner)
assert note.data_group_id == HOSTED
async def test_a_memory_is_recorded_in_the_group_and_forgotten_only_there(db, owner, chats):
home, away = chats
memories.add(db, owner=owner, content="Keep this at home.", group=DEFAULT_GROUP)
await _run(_context(db, owner, away), "memory_add", content="Hosted fact.")
assert [m.content for m in memories.all_for(db, owner, HOSTED)] == ["Hosted fact."]
await _run(_context(db, owner, away), "memory_forget", content="Keep this at home.")
assert [m.content for m in memories.all_for(db, owner, DEFAULT_GROUP)] == [
"Keep this at home."
]
# The same call from the memory's own group does forget it, so the refusal
# above is the group and not a mistyped argument.
await _run(_context(db, owner, home), "memory_forget", content="Keep this at home.")
db.expire_all()
assert memories.all_for(db, owner, DEFAULT_GROUP) == []
def test_the_same_fact_in_two_groups_is_two_memories(db, owner):
first = memories.add(db, owner=owner, content="Same.", group=DEFAULT_GROUP)
second = memories.add(db, owner=owner, content="Same.", group=HOSTED)
assert first.id != second.id
async def test_a_document_in_another_group_cannot_be_fetched_by_id(db, owner, chats):
home, away = chats
base = documents.create_base(db, owner=owner, name="Home")
document = documents.store_upload(
db, owner=owner, payload=b"The mallorn is golden.", filename="a.txt", base=base
)
context = _context(db, owner, away)
assert (await _run(context, "knowledge_search", query="mallorn")).event["results"] == []
assert (await _run(context, "knowledge_get", id=document.id)).event["status"] == "error"
async def test_a_report_in_another_group_cannot_be_read(db, owner, chats):
home, away = chats
report = reports.create(db, owner=owner, title="Home report", body="mallorn", unread=False)
context = _context(db, owner, away)
assert (await _run(context, "report_get", id=report.id)).event["status"] == "error"
async def test_a_skill_in_another_group_cannot_be_fetched(db, owner, chats):
home, away = chats
skills.create(db, owner=owner, name="home-skill", description="Home.", body="SECRET")
outcome = await _run(_context(db, owner, away), "skill_get", name="home-skill")
assert "SECRET" not in outcome.content
def test_a_skill_name_taken_in_another_group_says_so(db, owner):
skills.create(db, owner=owner, name="shared-name", description="d", body="b")
with pytest.raises(skills.SkillError, match="another data group"):
skills.create(
db, owner=owner, name="shared-name", description="d", body="b", group=HOSTED
)
async def test_an_impression_is_written_under_the_groups_key(db, owner, chats):
home, away = chats
await _run(_context(db, owner, away), "impression_write", content="Terse.")
row = db.scalar(select(Impression))
assert row.model_key == personas.key_for("cloud-model", HOSTED)
def test_each_group_gets_its_own_default_base(db, owner, chats):
home = documents.default_base(db, owner)
away = documents.default_base(db, owner, HOSTED)
assert home.id != away.id
assert away.data_group_id == HOSTED
assert home.name != away.name
+273
View File
@@ -0,0 +1,273 @@
"""The services that read a group's data, and the screens that arrange groups.
The embedder is sent the full text of everything it indexes and the reviewer is
sent every picture with its prompt, so a group can name its own of each and
falls back to the instance's when it names none. Then the three places groups
are arranged: the admin page, the person's own settings, and the library.
"""
from __future__ import annotations
import pytest
from sqlalchemy import select
from lembas.db.models import (
DEFAULT_GROUP,
Connection,
DataGroup,
Model,
Note,
User,
)
from lembas.services import data_groups, settings_store
from lembas.services import tools as tools_service
from lembas.services.crypto import encrypt
from lembas.services.images import tool as image_tool
from lembas.services.library import indexing, notes
HOSTED = "hosted"
@pytest.fixture
def owner(db, registered) -> User:
return db.scalars(select(User).order_by(User.created_at)).first()
@pytest.fixture
def setup(db, owner):
db.add(DataGroup(id=HOSTED, name="Hosted"))
local = Connection(name="Local", base_url="http://127.0.0.1:1", api_key_encrypted=encrypt(""))
cloud = Connection(
name="Cloud",
base_url="http://127.0.0.1:2",
api_key_encrypted=encrypt(""),
data_group_id=HOSTED,
)
db.add_all([local, cloud])
db.flush()
db.add_all(
[
Model(connection_id=local.id, model_id="local-embed",
capabilities_json={"embeddings": True}),
Model(connection_id=cloud.id, model_id="cloud-embed",
capabilities_json={"embeddings": True}),
Model(connection_id=local.id, model_id="local-eye",
capabilities_json={"vision": True}),
Model(connection_id=cloud.id, model_id="cloud-eye",
capabilities_json={"vision": True}),
]
)
db.commit()
return local, cloud
def _plain_user(db) -> User:
user = User(email="sam@shire.test", name="Sam", password_hash="x", role="user")
db.add(user)
db.commit()
return user
# --- The embedder -----------------------------------------------------------------
def test_a_group_without_its_own_embedder_uses_the_instances(db, setup):
settings_store.update(db, {"embedding_model_id": "local-embed"}, key=settings_store.EXTRACTION)
assert indexing.embedder(db, HOSTED).model_id == "local-embed"
def test_a_group_with_its_own_embedder_uses_that(db, setup):
settings_store.update(db, {"embedding_model_id": "local-embed"}, key=settings_store.EXTRACTION)
group = data_groups.get(db, HOSTED)
group.embedding_model_id = "cloud-embed"
db.commit()
assert indexing.embedder(db, HOSTED).model_id == "cloud-embed"
assert indexing.embedder(db, DEFAULT_GROUP).model_id == "local-embed"
def test_a_record_is_indexed_by_its_own_groups_embedder(db, owner, setup):
group = data_groups.get(db, HOSTED)
group.embedding_model_id = "cloud-embed"
db.commit()
note = notes.create(db, owner=owner, title="t", body="b", group=HOSTED)
assert indexing.group_of_row(db, note) == HOSTED
assert indexing.embedder(db, indexing.group_of_row(db, note)).model_id == "cloud-embed"
def test_any_configured_counts_a_groups_own_embedder(db, setup):
assert indexing.any_configured(db) is False
group = data_groups.get(db, HOSTED)
group.embedding_model_id = "cloud-embed"
db.commit()
assert indexing.any_configured(db) is True
# --- The reviewer -------------------------------------------------------------------
def test_the_reviewer_is_the_groups_own_when_it_names_one(db, setup):
group = data_groups.get(db, HOSTED)
group.review_model_id = "cloud-eye"
db.commit()
config = {"review_enabled": True, "review_model_id": "local-eye"}
hosted = tools_service.ToolContext(owner_id="x", image_config=config, data_group=HOSTED)
home = tools_service.ToolContext(owner_id="x", image_config=config)
assert image_tool._reviewer(hosted)[1] == "cloud-eye"
assert image_tool._reviewer(home)[1] == "local-eye"
# --- The admin page ---------------------------------------------------------------------
def test_the_admin_page_lists_groups_and_creates_one(client, db, setup):
assert "Hosted" in client.get("/admin/data-groups").text
response = client.post("/admin/data-groups", data={"name": "Work"}, follow_redirects=False)
assert response.status_code == 303
assert db.scalar(select(DataGroup).where(DataGroup.name == "Work")) is not None
def test_putting_a_connection_into_a_group_and_taking_it_out(client, db, setup):
local, cloud = setup
client.post(
f"/admin/data-groups/{HOSTED}",
data={"name": "Hosted", "connections_sent": "1", "connection_ids": [local.id]},
)
db.expire_all()
assert db.get(Connection, local.id).data_group_id == HOSTED
# Unticked: back to the default group, never to "no group".
assert db.get(Connection, cloud.id).data_group_id == DEFAULT_GROUP
def test_the_detail_page_flags_a_service_in_another_group(client, db, setup):
settings_store.update(db, {"embedding_model_id": "local-embed"}, key=settings_store.EXTRACTION)
page = client.get(f"/admin/data-groups/{HOSTED}").text
assert "its connection is in Default" in page
def test_deleting_a_group_in_use_is_refused_with_the_reason(client, db, setup):
response = client.post(f"/admin/data-groups/{HOSTED}/delete", follow_redirects=True)
assert "connections" in response.text
assert data_groups.get(db, HOSTED) is not None
def test_the_admin_page_is_for_administrators(client, db, setup, owner):
owner.role = "user"
db.commit()
assert client.get("/admin/data-groups").status_code in (303, 403, 404)
def test_the_connection_form_sets_the_group(client, db, setup):
local, _ = setup
client.post(
f"/admin/connections/{local.id}",
data={"name": "Local", "base_url": local.base_url, "data_group_id": HOSTED},
)
db.expire_all()
assert db.get(Connection, local.id).data_group_id == HOSTED
# --- The person's own settings -------------------------------------------------------------
def test_remapping_for_oneself_needs_the_permission(client, db, setup, owner):
local, _ = setup
owner.role = "user"
db.commit()
client.post("/api/preferences/data-groups", data={f"group__{local.id}": HOSTED})
db.expire_all()
assert data_groups.personal_map(db.get(User, owner.id)) == {}
settings_store.update(db, {"default_permissions": {data_groups.PERMISSION: True}})
client.post("/api/preferences/data-groups", data={f"group__{local.id}": HOSTED})
db.expire_all()
user = db.get(User, owner.id)
assert data_groups.personal_map(user) == {local.id: HOSTED}
assert data_groups.for_connection(db, user, local.id) == HOSTED
def test_a_personal_group_is_made_and_seen_only_by_its_owner(client, db, setup, owner):
client.post("/api/preferences/data-groups/new", data={"name": "Private"})
group = db.scalar(select(DataGroup).where(DataGroup.name == "Private"))
assert group.owner_id == owner.id
other = _plain_user(db)
assert group.id not in {g.id for g in data_groups.usable(db, other)}
def test_the_data_tab_appears_once_there_is_a_choice(client, db, setup):
page = client.get("/settings").text
assert 'id="tab-data"' in page
assert "Which group each connection reads" in page
# --- The library --------------------------------------------------------------------------------
def test_a_note_is_created_in_the_chosen_group(client, db, setup):
client.post("/api/library/notes", data={"title": "n", "body": "b", "data_group_id": HOSTED})
assert db.scalar(select(Note)).data_group_id == HOSTED
def test_a_group_the_person_may_not_use_falls_back_to_the_default(client, db, setup, owner):
other = _plain_user(db)
db.add(DataGroup(id="theirs", name="Theirs", owner_id=other.id))
db.commit()
client.post("/api/library/notes", data={"title": "n", "body": "b", "data_group_id": "theirs"})
assert db.scalar(select(Note)).data_group_id == DEFAULT_GROUP
def test_moving_a_note_needs_the_permission(client, db, setup, owner):
note = notes.create(db, owner=owner, title="n", body="b")
owner.role = "user"
db.commit()
client.post(f"/api/library/notes/{note.id}", data={"title": "n", "body": "b",
"data_group_id": HOSTED})
db.expire_all()
assert db.get(Note, note.id).data_group_id == DEFAULT_GROUP
settings_store.update(db, {"default_permissions": {data_groups.PERMISSION: True,
"library.use": True}})
client.post(f"/api/library/notes/{note.id}", data={"title": "n", "body": "b",
"data_group_id": HOSTED})
db.expire_all()
assert db.get(Note, note.id).data_group_id == HOSTED
def test_the_notes_list_filters_by_group(client, db, setup, owner):
notes.create(db, owner=owner, title="Home note", body="b")
notes.create(db, owner=owner, title="Hosted note", body="b", group=HOSTED)
page = client.get(f"/library/notes?group={HOSTED}").text
assert "Hosted note" in page and "Home note" not in page
def test_a_single_group_instance_shows_nothing_about_groups(client, db, owner):
"""The ordinary instance: no chip, no select, no tab -- for anybody who could
not make a personal group either. An administrator can, so the tab is theirs."""
notes.create(db, owner=owner, title="n", body="b")
owner.role = "user"
db.commit()
assert 'name="data_group_id"' not in client.get("/library/notes/new").text
assert 'id="tab-data"' not in client.get("/settings").text
# --- Two embedders of the same width -----------------------------------------------------
def test_a_query_skips_chunks_another_model_of_the_same_width_made(db, owner):
"""Width cannot tell two 1024-wide models apart, and with an embedder per
group two of them on one instance is ordinary. The query says which model
made it, and only that model's chunks are scored."""
from lembas.db.models import CHUNK_NOTE, Chunk
from lembas.services.library import chunks as chunk_service
from lembas.services.library import retrieval
ours = notes.create(db, owner=owner, title="Ours", body="x")
theirs = notes.create(db, owner=owner, title="Theirs", body="y")
for note, model_id, vector in ((ours, "embed-a", [0.6, 0.8]), (theirs, "embed-b", [1.0, 0.0])):
db.add(
Chunk(
owner_id=owner.id,
resource_type=CHUNK_NOTE,
resource_id=note.id,
ordinal=0,
text="t",
vector=chunk_service.pack(vector),
dims=2,
model_id=model_id,
)
)
db.commit()
query = retrieval.QueryVector([1.0, 0.0])
query.model_id = "embed-a"
assert [hit.id for hit in retrieval.semantic_ids(db, CHUNK_NOTE, query)] == [ours.id]
# A plain list keeps the old width-only behaviour, and the closer vector wins.
plain = retrieval.semantic_ids(db, CHUNK_NOTE, [1.0, 0.0])
assert [hit.id for hit in plain][0] == theirs.id
+293
View File
@@ -0,0 +1,293 @@
"""Data groups: how one is resolved, the startup sweep, deleting one, and upgrading.
What a group *isolates* is `test_data_group_isolation.py`; how a chat is pinned to
one is `test_data_group_chat_pin.py`. This file is the machinery underneath both:
the resolution order, which lives in one function and must keep living there, and
the upgrade from a database that has never heard of groups.
"""
from __future__ import annotations
import pytest
from sqlalchemy import inspect, select, text
from lembas.db.migrations import sync_schema
from lembas.db.models import (
DEFAULT_GROUP,
Chat,
Connection,
DataGroup,
Memory,
Model,
Note,
Persona,
User,
)
from lembas.db.session import get_engine
from lembas.services import data_groups, personas, settings_store
from lembas.services.crypto import encrypt
@pytest.fixture
def owner(db, registered) -> User:
return db.scalars(select(User).order_by(User.created_at)).first()
@pytest.fixture
def reader(db, registered) -> User:
"""A second account, not an administrator, so permissions actually apply."""
user = User(email="sam@shire.test", name="Sam", password_hash="x", role="user")
db.add(user)
db.commit()
return user
@pytest.fixture
def two(db) -> tuple[Connection, Connection]:
"""A local connection in the default group and a hosted one in its own."""
db.add(DataGroup(id="hosted", name="Hosted"))
local = Connection(name="Local", base_url="http://127.0.0.1:1", api_key_encrypted=encrypt(""))
cloud = Connection(
name="Cloud",
base_url="http://127.0.0.1:2",
api_key_encrypted=encrypt(""),
data_group_id="hosted",
)
db.add_all([local, cloud])
db.flush()
db.add_all(
[
Model(connection_id=local.id, model_id="local-model", position=0),
Model(connection_id=cloud.id, model_id="cloud-model", position=1),
]
)
db.commit()
return local, cloud
def _grant_manage(db, user: User) -> None:
settings_store.update(db, {"default_permissions": {data_groups.PERMISSION: True}})
# --- Resolution --------------------------------------------------------------------
def test_a_connection_with_no_group_is_in_the_default_one(db, two, reader):
local, _ = two
assert data_groups.for_connection(db, reader, local.id) == DEFAULT_GROUP
def test_the_administrators_choice_applies_to_everybody(db, two, reader):
_, cloud = two
assert data_groups.for_connection(db, reader, cloud.id) == "hosted"
assert data_groups.for_connection(db, None, cloud.id) == "hosted"
def test_a_personal_mapping_needs_the_permission(db, two, reader):
"""Stored and ignored without `data.manage`: taking the permission away puts a
person back on the instance's arrangement without anybody clearing anything."""
local, _ = two
reader.settings_json = {data_groups.SETTING_KEY: {local.id: "hosted"}}
db.commit()
assert data_groups.for_connection(db, reader, local.id) == DEFAULT_GROUP
_grant_manage(db, reader)
assert data_groups.for_connection(db, reader, local.id) == "hosted"
def test_a_mapping_to_somebody_elses_personal_group_is_ignored(db, two, reader, owner):
local, _ = two
db.add(DataGroup(id="theirs", name="Theirs", owner_id=owner.id))
reader.settings_json = {data_groups.SETTING_KEY: {local.id: "theirs"}}
db.commit()
_grant_manage(db, reader)
assert data_groups.for_connection(db, reader, local.id) == DEFAULT_GROUP
def test_a_connection_naming_a_deleted_group_falls_back_to_the_default(db, two, reader):
_, cloud = two
cloud.data_group_id = "gone"
db.commit()
assert data_groups.for_connection(db, reader, cloud.id) == DEFAULT_GROUP
def test_a_pair_without_a_connection_resolves_through_the_model(db, two, reader):
assert data_groups.for_pair(db, reader, "cloud-model") == "hosted"
assert data_groups.for_pair(db, reader, "local-model") == DEFAULT_GROUP
def test_a_chat_keeps_the_group_it_was_stamped_with(db, two, reader):
"""Derived once, then read. A model moved afterwards does not carry the chat."""
_, cloud = two
chat = Chat(user_id=reader.id, model_id="cloud-model", connection_id=cloud.id)
db.add(chat)
db.commit()
assert data_groups.for_chat(db, chat) == "hosted"
db.commit()
cloud.data_group_id = DEFAULT_GROUP
db.commit()
assert data_groups.for_chat(db, chat) == "hosted"
def test_usable_groups_are_the_instances_and_ones_own(db, two, reader, owner):
db.add_all(
[
DataGroup(id="mine", name="Mine", owner_id=reader.id),
DataGroup(id="not-mine", name="Not mine", owner_id=owner.id),
]
)
db.commit()
ids = {group.id for group in data_groups.usable(db, reader)}
assert ids == {DEFAULT_GROUP, "hosted", "mine"}
def test_one_group_means_there_is_nothing_to_choose(db, reader):
assert data_groups.several(db, reader) is False
db.add(DataGroup(id="second", name="Second"))
db.commit()
assert data_groups.several(db, reader) is True
# --- Deleting ------------------------------------------------------------------------
def test_the_default_group_cannot_be_deleted(db):
with pytest.raises(ValueError, match="default group"):
data_groups.delete(db, data_groups.ensure_default(db))
def test_a_group_with_records_in_it_cannot_be_deleted(db, reader):
group = DataGroup(id="busy", name="Busy")
db.add(group)
db.add(Note(owner_id=reader.id, title="n", body="b", data_group_id="busy"))
db.commit()
with pytest.raises(ValueError, match="1 notes"):
data_groups.delete(db, group)
def test_a_group_a_connection_is_in_cannot_be_deleted(db, two):
with pytest.raises(ValueError, match="connections"):
data_groups.delete(db, data_groups.get(db, "hosted"))
def test_deleting_an_empty_group_clears_every_mapping_to_it(db, reader):
group = DataGroup(id="empty", name="Empty")
db.add(group)
reader.settings_json = {data_groups.SETTING_KEY: {"some-connection": "empty"}}
db.commit()
data_groups.delete(db, group)
db.refresh(reader)
assert data_groups.personal_map(reader) == {}
assert data_groups.get(db, "empty") is None
# --- The sweep -------------------------------------------------------------------------
def test_the_sweep_puts_old_rows_in_the_default_group(db, reader):
db.add(Memory(owner_id=reader.id, content="old"))
db.commit()
assert db.scalar(select(Memory)).data_group_id is None
data_groups.sweep_unassigned(db)
assert db.scalar(select(Memory)).data_group_id == DEFAULT_GROUP
def test_the_sweep_gives_a_chat_its_models_group(db, two, reader):
"""Not simply the default: a chat some path created without stamping one
belongs where its model is, or its next turn is refused as a moved chat."""
chat = Chat(user_id=reader.id, model_id="cloud-model")
db.add(chat)
db.commit()
data_groups.sweep_unassigned(db)
db.refresh(chat)
assert chat.data_group_id == "hosted"
def test_a_row_the_sweep_has_not_reached_still_counts_as_default(db, reader):
db.add(Note(owner_id=reader.id, title="old", body=""))
db.commit()
found = db.scalars(select(Note).where(data_groups.condition(Note, DEFAULT_GROUP))).all()
assert [note.title for note in found] == ["old"]
# --- Personalities: namespaced, because the constraint cannot change ------------------
def test_the_default_group_keeps_the_bare_model_id():
assert personas.key_for("gpt-oss", DEFAULT_GROUP) == "gpt-oss"
assert personas.key_for("gpt-oss", None) == "gpt-oss"
def test_another_group_gets_its_own_key_and_splits_back():
key = personas.key_for("gpt-oss", "hosted")
assert key != "gpt-oss"
assert personas.split_key(key) == ("gpt-oss", "hosted")
assert personas.split_key("gpt-oss") == ("gpt-oss", DEFAULT_GROUP)
def test_a_group_falls_back_to_the_administrators_default(db, reader):
"""The admin default is keyed bare and reaches every group, until the model has
written one of its own with this person in that group."""
db.add(Persona(model_key="gpt-oss", owner_id=None, content="the default"))
db.commit()
key = personas.key_for("gpt-oss", "hosted")
assert personas.block(db, key, reader) == "the default"
personas.write(db, model_key=key, owner=reader, content="hosted self", author="model")
assert personas.block(db, key, reader) == "hosted self"
assert personas.block(db, "gpt-oss", reader) == "the default"
# --- Upgrading from 1.9.1 -----------------------------------------------------------------
# What 1.10.0 added: one table, and one column on each of these. Taken from the
# models rather than invented, and `test_the_recorded_shape_is_still_real`
# below is what stops the list rotting.
NEW_TABLE = "data_groups"
GROUPED_TABLES = (
"chats",
"connections",
"knowledge_bases",
"memories",
"notes",
"reports",
"schedules",
"skills",
)
def _rollback_to_1_9_1(engine) -> None:
with engine.begin() as connection:
connection.execute(text(f"DROP TABLE IF EXISTS {NEW_TABLE}"))
for table in GROUPED_TABLES:
connection.execute(text(f"ALTER TABLE {table} DROP COLUMN data_group_id"))
def test_the_recorded_shape_is_still_real():
from lembas.db.base import Base
grouped = {
table.name for table in Base.metadata.sorted_tables if "data_group_id" in table.columns
}
assert grouped == set(GROUPED_TABLES)
def test_a_1_9_1_database_with_data_upgrades(db, reader):
engine = get_engine()
db.add(Memory(owner_id=reader.id, content="kept"))
db.add(Persona(model_key="gpt-oss", owner_id=reader.id, content="mine"))
db.commit()
db.close()
_rollback_to_1_9_1(engine)
assert "data_group_id" not in {c["name"] for c in inspect(engine).get_columns("memories")}
changes = sync_schema(engine)
assert any(NEW_TABLE in change for change in changes)
for table in GROUPED_TABLES:
assert "data_group_id" in {c["name"] for c in inspect(engine).get_columns(table)}
# A nullable column arrives empty; the sweep is what files it.
memory = db.scalar(select(Memory))
assert memory.data_group_id is None
data_groups.sweep_unassigned(db)
db.refresh(memory)
assert memory.data_group_id == DEFAULT_GROUP
# The person's personality was keyed bare, and the default group still reads it.
user = db.get(User, reader.id)
assert personas.block(db, personas.key_for("gpt-oss", DEFAULT_GROUP), user) == "mine"
assert data_groups.get(db, DEFAULT_GROUP) is not None