Every injected prompt becomes editable, and several get written
The instructions LLeMbas puts in front of a model were hard-coded: six
strings in a GUIDANCE dict, two headings, and the title request inline in
chat.py. An operator could not see what was being sent, let alone change
it, and there was nowhere for a custom tool to contribute its own guidance
when custom tools land.
services/prompts.py now holds each piece as a Fragment, and /admin/prompts
edits them with a preview of the whole assembled system message including
unsaved edits. harness.py keeps only the decisions -- which fragments apply
to this request, and what their variables resolve to.
The design turns on one choice: a fragment carries its gate as data
(families, requires, when_tools) rather than as a callable, because a
database row can carry the same three fields. Custom tools will therefore
register a fragment source and change nothing else -- there is a test that
says exactly that, and it is the reason the rest of the shape is what it is.
Consequences worth knowing:
- Defaults live in code, overrides in the database, and text equal to its
default is never stored. Otherwise pressing Save once would freeze
today's wording forever and no later release could improve it.
- An empty override means off. A fragment that was not submitted at all
keeps what it had, because it may be missing from the page only because
whatever contributes it is currently switched off.
- requires= replaced the hand-written pair of memory guidance variants.
The sentence that refers to a section now lives inside that section, so
it cannot outlive it. That was the general problem the pair was a
special case of.
- {{name}}, with anything unrecognised passing through verbatim. The name
grammar is the guard: {"total": 1} and ${PATH} are not candidates.
Substitution is one pass and never recursive, because {{memories}}
carries text a model wrote.
The wording is also overhauled, and a model now gets the core fragments
even with no tools -- the date above all. "An empty harness is worse than
none" was about tokens that say nothing; a model with no clock being asked
about the present is not that. Clearing those boxes restores the old
silence exactly. New: today's date, who it is talking to, the three-round
tool budget, that tool results are not replayed, that anything a tool
returns is data rather than instruction, and what the <document> wrapper
around an attachment is. Extended: memory_forget, notes_edit/delete,
skill_create/edit, and reading a knowledge document in full rather than
answering from an extract.
Tool descriptions stay in code and are listed read-only. They are schema
and they state facts about what a runner does; an edit would make the text
a lie with nothing to catch it.
No schema change -- one JSON row in the settings table.
488 tests. Version 0.2.0, which also invalidates the service worker cache
so the green artwork appears without a hard reload.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,147 @@
|
||||
"""The prompt editor: what it saves, what it refuses to save, and the preview."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from lembas.db.models import User
|
||||
from lembas.services import harness, prompts, settings_store
|
||||
from lembas.services.library import memories as memories_service
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def plain_user(client: TestClient, db, registered) -> User:
|
||||
"""A second, non-admin account. Leaves the client signed in as them."""
|
||||
client.post("/auth/logout", follow_redirects=False)
|
||||
client.post(
|
||||
"/auth/register",
|
||||
data={"name": "Sam", "email": "sam@shire.test", "password": "potatoes-po-ta-toes"},
|
||||
follow_redirects=False,
|
||||
)
|
||||
return db.scalar(select(User).where(User.email == "sam@shire.test"))
|
||||
|
||||
|
||||
def _owner(db) -> User:
|
||||
return db.scalar(select(User).where(User.email == "frodo@shire.test"))
|
||||
|
||||
|
||||
# --- Access ------------------------------------------------------------------
|
||||
def test_the_page_lists_every_fragment(client: TestClient, registered):
|
||||
page = client.get("/admin/prompts").text
|
||||
for fragment in prompts.BUILTIN:
|
||||
assert f'name="prompt.{fragment.key}"' in page, fragment.key
|
||||
|
||||
|
||||
def test_the_page_is_refused_to_a_plain_user(client: TestClient, plain_user):
|
||||
assert client.get("/admin/prompts").status_code == 403
|
||||
assert client.post("/admin/prompts", data={}).status_code == 403
|
||||
|
||||
|
||||
# --- Saving ------------------------------------------------------------------
|
||||
def test_saving_changes_what_a_model_is_told(client: TestClient, db, registered):
|
||||
client.post(
|
||||
"/admin/prompts",
|
||||
data={"prompt.tool.web_search": "- Always search twice.", "max_harness_chars": "0"},
|
||||
follow_redirects=False,
|
||||
)
|
||||
text = harness.compose(db, _owner(db), _tools("web_search"))
|
||||
assert "- Always search twice." in text
|
||||
assert "Look things up" not in text
|
||||
|
||||
|
||||
def test_a_cleared_box_turns_the_fragment_off(client: TestClient, db, registered):
|
||||
"""The trap this page was written around: FastAPI cannot tell an empty form
|
||||
field from an absent one, so the handler reads the raw form."""
|
||||
client.post(
|
||||
"/admin/prompts",
|
||||
data={"prompt.core.style": "", "max_harness_chars": "0"},
|
||||
follow_redirects=False,
|
||||
)
|
||||
assert prompts.stored(db) == {"core.style": ""}
|
||||
assert "Answer in the language" not in harness.compose(db, _owner(db), [])
|
||||
|
||||
|
||||
def test_a_fragment_not_submitted_at_all_is_left_alone(client: TestClient, db, registered):
|
||||
prompts.save(db, {"core.heading": "## Rules"})
|
||||
client.post("/admin/prompts", data={"max_harness_chars": "0"}, follow_redirects=False)
|
||||
assert prompts.resolve(db, "core.heading") == "## Rules"
|
||||
|
||||
|
||||
def test_saving_the_built_in_wording_stores_nothing(client: TestClient, db, registered):
|
||||
"""Opening the page and pressing Save must not freeze today's defaults, or a
|
||||
later release could never improve them."""
|
||||
page_fields = {f"prompt.{f.key}": f.default for f in prompts.BUILTIN}
|
||||
client.post(
|
||||
"/admin/prompts", data={**page_fields, "max_harness_chars": "0"}, follow_redirects=False
|
||||
)
|
||||
assert prompts.stored(db) == {}
|
||||
|
||||
|
||||
def test_the_character_cap_is_clamped_and_kept(client: TestClient, db, registered):
|
||||
client.post("/admin/prompts", data={"max_harness_chars": "-5"}, follow_redirects=False)
|
||||
assert settings_store.get(db, "max_harness_chars", key=settings_store.PROMPTS) == 0
|
||||
|
||||
client.post("/admin/prompts", data={"max_harness_chars": "600"}, follow_redirects=False)
|
||||
assert harness.limit_for(db) == 600
|
||||
|
||||
|
||||
# --- Resetting ---------------------------------------------------------------
|
||||
def test_use_default_fills_the_box_without_saving(client: TestClient, db, registered):
|
||||
prompts.save(db, {"core.heading": "## Rules"})
|
||||
response = client.post("/admin/prompts/default", data={"key": "core.heading"})
|
||||
|
||||
assert "## How to work" in response.text
|
||||
assert "edited" not in response.text
|
||||
# Nothing was written: it takes a Save to make it stick.
|
||||
assert prompts.resolve(db, "core.heading") == "## Rules"
|
||||
|
||||
|
||||
def test_use_default_on_an_unknown_key_is_a_404(client: TestClient, registered):
|
||||
assert client.post("/admin/prompts/default", data={"key": "nope.nope"}).status_code == 404
|
||||
|
||||
|
||||
def test_restore_all_defaults_empties_the_overrides(client: TestClient, db, registered):
|
||||
prompts.save(db, {"core.heading": "## Rules", "core.style": ""})
|
||||
client.post("/admin/prompts/reset", follow_redirects=False)
|
||||
assert prompts.stored(db) == {}
|
||||
|
||||
|
||||
# --- Preview -----------------------------------------------------------------
|
||||
def test_the_preview_shows_text_that_has_not_been_saved(client: TestClient, db, registered):
|
||||
body = client.post(
|
||||
"/admin/prompts/preview",
|
||||
data={"prompt.core.heading": "## Draft heading", "preview_family": ["web_search"]},
|
||||
).text
|
||||
|
||||
assert "## Draft heading" in body
|
||||
assert prompts.stored(db) == {}
|
||||
|
||||
|
||||
def test_the_preview_only_shows_guidance_for_the_families_ticked(client: TestClient, registered):
|
||||
body = client.post("/admin/prompts/preview", data={"preview_family": ["notes"]}).text
|
||||
assert "You keep notes" in body
|
||||
assert "Look things up" not in body
|
||||
|
||||
|
||||
def test_the_preview_escapes_what_a_model_wrote(client: TestClient, db, registered):
|
||||
"""A memory is model-written text on an admin page. Hard rule 6 applies to
|
||||
the preview exactly as it does to a chat bubble."""
|
||||
memories_service.add(db, owner=_owner(db), content="<img src=x onerror=alert(1)>")
|
||||
body = client.post("/admin/prompts/preview", data={"preview_family": ["memory"]}).text
|
||||
|
||||
assert "<img src=x" not in body
|
||||
assert "<img src=x" in body
|
||||
|
||||
|
||||
def test_the_preview_warns_when_the_cap_would_cut_it_off(client: TestClient, db, registered):
|
||||
settings_store.update(db, {"max_harness_chars": 60}, key=settings_store.PROMPTS)
|
||||
body = client.post("/admin/prompts/preview", data={"preview_family": ["web_search"]}).text
|
||||
assert "past the cap is cut off" in body
|
||||
|
||||
|
||||
def _tools(*names):
|
||||
from lembas.services import tools as tools_service
|
||||
|
||||
return [tools_service.REGISTRY[name].schema for name in names]
|
||||
+46
-2
@@ -2,6 +2,8 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
@@ -131,6 +133,44 @@ def test_fallback_title_of_nothing():
|
||||
assert chat_service.fallback_title(" ") == "New chat"
|
||||
|
||||
|
||||
async def test_the_title_prompt_carries_the_exchange(mock_http):
|
||||
"""The wording is a fragment an administrator can edit, so what reaches the
|
||||
endpoint has to be the substituted text, not the template."""
|
||||
seen: list[str] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
seen.append(json.loads(request.content)["messages"][0]["content"])
|
||||
return httpx.Response(200, json={"choices": [{"message": {"content": "A short name"}}]})
|
||||
|
||||
mock_http(handler)
|
||||
endpoint = Endpoint("http://x.test", "", {})
|
||||
title = await chat_service.generate_title(
|
||||
endpoint,
|
||||
"m",
|
||||
"What is lembas?",
|
||||
"Waybread.",
|
||||
template="Name this: {{question}} / {{answer}} / {{nonsense}}",
|
||||
)
|
||||
|
||||
assert title == "A short name"
|
||||
assert seen == ["Name this: What is lembas? / Waybread. / {{nonsense}}"]
|
||||
|
||||
|
||||
async def test_an_empty_title_prompt_asks_no_model_at_all(mock_http):
|
||||
"""Clearing the fragment is how auto-titling is turned off. It must not
|
||||
cost a request that is then thrown away."""
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response: # pragma: no cover - must not run
|
||||
raise AssertionError("the endpoint was contacted")
|
||||
|
||||
mock_http(handler)
|
||||
endpoint = Endpoint("http://x.test", "", {})
|
||||
title = await chat_service.generate_title(
|
||||
endpoint, "m", "What is lembas?", "Waybread.", template=" "
|
||||
)
|
||||
assert title == "What is lembas?"
|
||||
|
||||
|
||||
# --- Chats and folders (through the API) -------------------------------------
|
||||
def _add_connection(db) -> Connection:
|
||||
# Port 1 refuses connections, which is what the error-path test relies on.
|
||||
@@ -316,7 +356,8 @@ def test_history_skips_failed_and_empty_turns(db, user_id):
|
||||
)
|
||||
db.commit()
|
||||
|
||||
contents = [m["content"] for m in chat_service.build_request(db, chat)["messages"]]
|
||||
messages = chat_service.build_request(db, chat)["messages"]
|
||||
contents = [m["content"] for m in messages if m["role"] != "system"]
|
||||
assert contents == ["one", "two"]
|
||||
|
||||
|
||||
@@ -332,7 +373,10 @@ def test_system_prompt_leads_the_message_list(db, user_id):
|
||||
db.commit()
|
||||
|
||||
messages = chat_service.build_request(db, chat)["messages"]
|
||||
assert messages[0] == {"role": "system", "content": "You are terse."}
|
||||
assert messages[0]["role"] == "system"
|
||||
# The harness precedes it inside the same message; the authored prompt is
|
||||
# last, where it is closest to the conversation.
|
||||
assert messages[0]["content"].endswith("You are terse.")
|
||||
|
||||
|
||||
def test_streaming_reports_an_unreachable_endpoint_in_the_thread(
|
||||
|
||||
+18
-8
@@ -17,6 +17,16 @@ from lembas.services import settings_store
|
||||
from lembas.services.crypto import encrypt
|
||||
|
||||
|
||||
def turns(body: dict) -> list[dict]:
|
||||
"""The conversation turns, without the system preamble in front of them.
|
||||
|
||||
`build_request` always emits a system message now -- the harness carries the
|
||||
date even for a model with no tools -- so a test about a *user* turn has to
|
||||
say which turn it means rather than assume index 0.
|
||||
"""
|
||||
return [message for message in body["messages"] if message["role"] != "system"]
|
||||
|
||||
|
||||
# --- Fixtures ----------------------------------------------------------------
|
||||
def png_bytes(width: int = 40, height: int = 30, mode: str = "RGB") -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
@@ -365,7 +375,7 @@ def test_images_become_multimodal_parts_for_a_vision_model(client: TestClient, d
|
||||
|
||||
chat = db.get(Chat, chat_with_model)
|
||||
payload = chat_service.build_request(db, chat)
|
||||
content = payload["messages"][0]["content"]
|
||||
content = turns(payload)[0]["content"]
|
||||
|
||||
assert isinstance(content, list)
|
||||
assert content[0] == {"type": "text", "text": "what is this?"}
|
||||
@@ -387,7 +397,7 @@ def test_images_are_withheld_from_a_model_without_vision(client: TestClient, db,
|
||||
)
|
||||
|
||||
chat = db.get(Chat, chat_with_model)
|
||||
content = chat_service.build_request(db, chat)["messages"][0]["content"]
|
||||
content = turns(chat_service.build_request(db, chat))[0]["content"]
|
||||
assert isinstance(content, str)
|
||||
assert content == "what is this?"
|
||||
|
||||
@@ -396,7 +406,7 @@ def test_a_plain_turn_stays_a_plain_string(client: TestClient, db, chat_with_mod
|
||||
"""The list form is a reliable 400 from endpoints that do not implement it."""
|
||||
client.post(f"/api/chats/{chat_with_model}/messages", data={"content": "just words"})
|
||||
chat = db.get(Chat, chat_with_model)
|
||||
assert chat_service.build_request(db, chat)["messages"][0]["content"] == "just words"
|
||||
assert turns(chat_service.build_request(db, chat))[0]["content"] == "just words"
|
||||
|
||||
|
||||
def test_document_text_is_prepended_in_tags(client: TestClient, db, chat_with_model):
|
||||
@@ -410,7 +420,7 @@ def test_document_text_is_prepended_in_tags(client: TestClient, db, chat_with_mo
|
||||
)
|
||||
|
||||
chat = db.get(Chat, chat_with_model)
|
||||
content = chat_service.build_request(db, chat)["messages"][0]["content"]
|
||||
content = turns(chat_service.build_request(db, chat))[0]["content"]
|
||||
assert '<document name="report.txt">' in content
|
||||
assert "Quarterly results were good." in content
|
||||
# The question comes after the material it refers to.
|
||||
@@ -430,7 +440,7 @@ def test_documents_reach_a_model_without_vision(client: TestClient, db, chat_wit
|
||||
)
|
||||
|
||||
chat = db.get(Chat, chat_with_model)
|
||||
content = chat_service.build_request(db, chat)["messages"][0]["content"]
|
||||
content = turns(chat_service.build_request(db, chat))[0]["content"]
|
||||
assert "important detail" in content
|
||||
|
||||
|
||||
@@ -445,7 +455,7 @@ def test_truncation_is_declared_to_the_model(client: TestClient, db, chat_with_m
|
||||
)
|
||||
|
||||
chat = db.get(Chat, chat_with_model)
|
||||
assert "(truncated)" in chat_service.build_request(db, chat)["messages"][0]["content"]
|
||||
assert "(truncated)" in turns(chat_service.build_request(db, chat))[0]["content"]
|
||||
|
||||
|
||||
def test_an_attachment_only_turn_still_reaches_the_model(client: TestClient, db, chat_with_model):
|
||||
@@ -456,7 +466,7 @@ def test_an_attachment_only_turn_still_reaches_the_model(client: TestClient, db,
|
||||
)
|
||||
|
||||
chat = db.get(Chat, chat_with_model)
|
||||
messages = chat_service.build_request(db, chat)["messages"]
|
||||
messages = turns(chat_service.build_request(db, chat))
|
||||
assert len(messages) == 1
|
||||
assert messages[0]["content"][0]["type"] == "image_url"
|
||||
|
||||
@@ -587,6 +597,6 @@ def test_a_browser_serialising_the_form_actually_sends_the_attachment(
|
||||
assert attachment.message_id == message.id
|
||||
|
||||
chat = db.get(Chat, chat_with_model)
|
||||
content = chat_service.build_request(db, chat)["messages"][0]["content"]
|
||||
content = turns(chat_service.build_request(db, chat))[0]["content"]
|
||||
assert isinstance(content, list), "the image never reached the model"
|
||||
assert any(p.get("type") == "image_url" for p in content)
|
||||
|
||||
+73
-8
@@ -7,7 +7,7 @@ import pytest
|
||||
from lembas.db.models import Chat, Connection, Model, User
|
||||
from lembas.security.passwords import hash_password
|
||||
from lembas.services import chat as chat_service
|
||||
from lembas.services import harness, settings_store
|
||||
from lembas.services import harness, prompts, settings_store
|
||||
from lembas.services import tools as tools_service
|
||||
from lembas.services.library import memories as memories_service
|
||||
from lembas.services.library import skills as skills_service
|
||||
@@ -26,11 +26,25 @@ def _tools(*names):
|
||||
|
||||
|
||||
# --- Composition -------------------------------------------------------------
|
||||
def test_no_tools_means_no_harness(db, owner):
|
||||
"""An empty harness is worse than none: tokens that say only that there is
|
||||
nothing to say."""
|
||||
def test_no_tools_means_no_tool_guidance(db, owner):
|
||||
"""The core fragments still go out -- a model with no tools has no clock
|
||||
either, and telling it the date is not "tokens that say nothing". What it
|
||||
must not get is instructions about tools it was never offered."""
|
||||
text = harness.compose(db, owner, [])
|
||||
assert "Today is" in text
|
||||
assert "You have tools" not in text
|
||||
assert "Look things up" not in text
|
||||
assert harness.compose(db, owner, None) == text
|
||||
|
||||
|
||||
def test_clearing_the_core_fragments_restores_an_empty_harness(db, owner):
|
||||
"""The behaviour change is a default, not a rule: an administrator who wants
|
||||
nothing sent to a tool-less model can still have exactly that."""
|
||||
prompts.save(
|
||||
db,
|
||||
{f.key: "" for f in prompts.BUILTIN if f.group == prompts.GROUP_CORE},
|
||||
)
|
||||
assert harness.compose(db, owner, []) == ""
|
||||
assert harness.compose(db, owner, None) == ""
|
||||
|
||||
|
||||
def test_only_the_guidance_for_offered_tools_appears(db, owner):
|
||||
@@ -123,11 +137,11 @@ def _system(body):
|
||||
return first.get("content", "") if first.get("role") == "system" else ""
|
||||
|
||||
|
||||
def test_a_request_without_tools_has_no_harness_and_no_tools_key(db, owner):
|
||||
def test_a_request_without_tools_carries_no_tool_guidance_and_no_tools_key(db, owner):
|
||||
chat = _chat(db, owner, capabilities={})
|
||||
body = chat_service.build_request(db, chat, tools=[], user=owner)
|
||||
assert "tools" not in body
|
||||
assert "How to work" not in _system(body)
|
||||
assert "You have tools" not in _system(body)
|
||||
|
||||
|
||||
def test_the_harness_precedes_the_authored_prompt(db, owner):
|
||||
@@ -162,7 +176,58 @@ def test_a_model_prompt_wins_when_the_chat_has_none(db, owner):
|
||||
settings_store.update(db, {"system_prompt": "Instance."})
|
||||
chat = _chat(db, owner, capabilities={}, model_prompt="Model.")
|
||||
system = _system(chat_service.build_request(db, chat, user=owner))
|
||||
assert system == "Model."
|
||||
assert system.endswith("Model.")
|
||||
assert "Instance." not in system
|
||||
assert chat_service.effective_system_prompt(db, chat) == "Model."
|
||||
|
||||
|
||||
def test_the_seam_line_only_appears_when_there_is_something_to_hand_over_to(db, owner):
|
||||
"""It introduces the authored prompt. With no authored prompt it would be
|
||||
pointing at nothing, which is the failure the whole `requires` idea exists
|
||||
to avoid."""
|
||||
bare = _chat(db, owner, capabilities={})
|
||||
assert "was written by whoever set up" not in _system(
|
||||
chat_service.build_request(db, bare, user=owner)
|
||||
)
|
||||
|
||||
authored = _chat(db, owner, capabilities={}, chat_prompt="Speak as Gandalf.")
|
||||
assert "was written by whoever set up" in _system(
|
||||
chat_service.build_request(db, authored, user=owner)
|
||||
)
|
||||
|
||||
|
||||
def test_an_administrators_wording_replaces_the_default(db, owner):
|
||||
prompts.save(db, {"core.today": "The date is {{today}}, more or less."})
|
||||
text = harness.compose(db, owner, [])
|
||||
assert "more or less." in text
|
||||
assert "Your training data stops well before this" not in text
|
||||
|
||||
|
||||
def test_the_attached_files_are_named_and_explained(db, owner):
|
||||
from lembas.db.models import Attachment
|
||||
|
||||
chat = _chat(db, owner, capabilities={})
|
||||
db.add(
|
||||
Attachment(
|
||||
user_id=owner.id,
|
||||
chat_id=chat.id,
|
||||
filename="report.txt",
|
||||
stored_name="x.txt",
|
||||
media_type="text/plain",
|
||||
size_bytes=10,
|
||||
kind="text",
|
||||
)
|
||||
)
|
||||
db.commit()
|
||||
|
||||
text = harness.compose(db, owner, [], chat)
|
||||
assert "report.txt" in text
|
||||
assert "<document name=" in text
|
||||
|
||||
|
||||
def test_a_chat_with_no_attachments_says_nothing_about_documents(db, owner):
|
||||
chat = _chat(db, owner, capabilities={})
|
||||
assert "<document name=" not in harness.compose(db, owner, [], chat)
|
||||
|
||||
|
||||
def test_the_tools_array_rides_along(db, owner):
|
||||
|
||||
@@ -425,7 +425,10 @@ def test_layers_replace_rather_than_stack(db, user_id):
|
||||
assert "INSTANCE" not in chat_service.effective_system_prompt(db, chat)
|
||||
|
||||
|
||||
def test_no_prompt_anywhere_sends_no_system_message(db, user_id):
|
||||
def test_no_prompt_anywhere_sends_no_authored_prompt(db, user_id):
|
||||
"""A system message still goes out -- the harness carries the date -- but
|
||||
nothing an administrator or the user wrote is in it, and there is no seam
|
||||
line introducing a prompt that does not exist."""
|
||||
connection = _connection(db)
|
||||
db.add(Model(connection_id=connection.id, model_id="m"))
|
||||
db.commit()
|
||||
@@ -433,7 +436,11 @@ def test_no_prompt_anywhere_sends_no_system_message(db, user_id):
|
||||
chat = Chat(user_id=user_id, model_id="m", connection_id=connection.id)
|
||||
db.add(chat)
|
||||
db.commit()
|
||||
assert chat_service.build_request(db, chat)["messages"] == []
|
||||
|
||||
messages = chat_service.build_request(db, chat)["messages"]
|
||||
assert [m for m in messages if m["role"] != "system"] == []
|
||||
assert chat_service.effective_system_prompt(db, chat) == ""
|
||||
assert "---" not in messages[0]["content"]
|
||||
|
||||
|
||||
# --- Admin bulk actions ------------------------------------------------------
|
||||
|
||||
@@ -0,0 +1,250 @@
|
||||
"""Prompt fragments: their variables, their gates, and what gets stored."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import pytest
|
||||
|
||||
from lembas.services import prompts, settings_store
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def extra_source():
|
||||
"""Register a fragment source for one test, then take it away again.
|
||||
|
||||
The registry is module state, so a test that adds to it and does not clean up
|
||||
leaks into every test after it.
|
||||
"""
|
||||
added: list = []
|
||||
|
||||
def install(*fragments: prompts.Fragment):
|
||||
added.extend(fragments)
|
||||
prompts.register_source(lambda db: fragments)
|
||||
|
||||
yield install
|
||||
prompts._SOURCES[:] = [prompts._builtin_source]
|
||||
|
||||
|
||||
# --- Substitution ------------------------------------------------------------
|
||||
def test_a_known_variable_is_replaced():
|
||||
assert prompts.substitute("Hello {{user_name}}.", {"user_name": "Frodo"}) == "Hello Frodo."
|
||||
|
||||
|
||||
def test_whitespace_inside_the_braces_is_allowed():
|
||||
assert prompts.substitute("{{ user_name }}", {"user_name": "Frodo"}) == "Frodo"
|
||||
|
||||
|
||||
def test_an_unknown_name_passes_through_exactly_as_typed():
|
||||
"""The fallback the whole syntax choice rests on: a collision with real
|
||||
prompt text degrades to "you get what you wrote"."""
|
||||
text = 'Reply as {"total": 1}, not {{Foo}} or {{a-b}} or ${PATH} or {{unknown}}.'
|
||||
assert prompts.substitute(text, {"user_name": "Frodo"}) == text
|
||||
|
||||
|
||||
def test_a_known_but_empty_variable_becomes_nothing():
|
||||
"""Not a pass-through. Pass-through is for names that are not variables, not
|
||||
for variables that happen to be empty -- otherwise an account with no name
|
||||
would send the literal braces to the model."""
|
||||
assert prompts.substitute("Talking to {{user_name}}.", {"user_name": ""}) == "Talking to ."
|
||||
|
||||
|
||||
def test_adjacent_variables_both_expand():
|
||||
assert prompts.substitute("{{a}}{{b}}", {"a": "1", "b": "2"}) == "12"
|
||||
|
||||
|
||||
def test_a_substituted_value_is_never_rescanned():
|
||||
"""A security property, not an accident: {{memories}} carries text a model
|
||||
wrote, and a memory reading "{{skills}}" must not pull in the skill index."""
|
||||
assert prompts.substitute("{{memories}}", {"memories": "{{skills}}", "skills": "SECRET"}) == (
|
||||
"{{skills}}"
|
||||
)
|
||||
|
||||
|
||||
def test_a_line_that_was_only_a_variable_disappears():
|
||||
"""So an empty value leaves no hole and no stranded heading."""
|
||||
assert prompts.substitute("before\n{{memories}}\nafter", {"memories": ""}) == "before\nafter"
|
||||
|
||||
|
||||
def test_a_line_with_no_variable_is_left_alone_even_when_blank():
|
||||
assert prompts.substitute("a\n\nb", {}) == "a\n\nb"
|
||||
|
||||
|
||||
# --- Gates -------------------------------------------------------------------
|
||||
def _assembled(db, **kwargs):
|
||||
return prompts.assemble(db, groups=(prompts.GROUP_CORE, prompts.GROUP_TOOLS), **kwargs)
|
||||
|
||||
|
||||
def test_requires_skips_the_whole_fragment(db, extra_source):
|
||||
extra_source(
|
||||
prompts.Fragment(
|
||||
key="core.zz_test",
|
||||
label="t",
|
||||
group=prompts.GROUP_CORE,
|
||||
order=900,
|
||||
default="Bases: {{knowledge_bases}}",
|
||||
requires=("knowledge_bases",),
|
||||
)
|
||||
)
|
||||
assert "Bases:" not in _assembled(db, variables={"knowledge_bases": ""})
|
||||
assert "Bases: contracts" in _assembled(db, variables={"knowledge_bases": "contracts"})
|
||||
|
||||
|
||||
def test_families_gate_a_fragment(db, extra_source):
|
||||
extra_source(
|
||||
prompts.Fragment(
|
||||
key="tool.zz_test",
|
||||
label="t",
|
||||
group=prompts.GROUP_TOOLS,
|
||||
order=900,
|
||||
default="- gated",
|
||||
families=("web_search",),
|
||||
)
|
||||
)
|
||||
assert "- gated" not in _assembled(db, variables={}, families=("notes",))
|
||||
assert "- gated" in _assembled(db, variables={}, families=("web_search",))
|
||||
|
||||
|
||||
def test_when_tools_gates_a_fragment(db):
|
||||
text = _assembled(db, variables={}, has_tools=False)
|
||||
assert "You have tools" not in text
|
||||
assert "You have tools" in _assembled(db, variables={}, has_tools=True)
|
||||
|
||||
|
||||
def test_a_run_of_bullets_stays_a_single_list(db):
|
||||
"""Five guidance fragments are five lines, not eleven."""
|
||||
text = prompts.assemble(
|
||||
db,
|
||||
groups=(prompts.GROUP_TOOLS,),
|
||||
variables={"memory_limit": "400"},
|
||||
families=("web_search", "notes"),
|
||||
)
|
||||
assert "\n\n- " not in text
|
||||
assert text.count("\n- ") == 1
|
||||
|
||||
|
||||
def test_assembly_is_capped(db):
|
||||
assert len(_assembled(db, variables={}, has_tools=True, limit=100)) <= 102
|
||||
|
||||
|
||||
# --- Storage -----------------------------------------------------------------
|
||||
def test_an_override_wins_over_the_default(db):
|
||||
prompts.save(db, {"core.heading": "## Rules"})
|
||||
assert prompts.resolve(db, "core.heading") == "## Rules"
|
||||
assert prompts.is_overridden(db, "core.heading")
|
||||
|
||||
|
||||
def test_an_empty_override_turns_a_fragment_off(db):
|
||||
prompts.save(db, {"core.style": ""})
|
||||
assert "Answer in the language" not in _assembled(db, variables={})
|
||||
|
||||
|
||||
def test_text_equal_to_the_default_is_not_stored(db):
|
||||
"""So that improving a default in a later release still reaches an instance
|
||||
whose administrator opened the page and pressed Save."""
|
||||
default = prompts.catalogue(db)["core.heading"].default
|
||||
prompts.save(db, {"core.heading": default})
|
||||
assert prompts.stored(db) == {}
|
||||
assert not prompts.is_overridden(db, "core.heading")
|
||||
|
||||
|
||||
def test_the_line_endings_a_browser_submits_do_not_count_as_an_edit(db):
|
||||
"""A textarea posts CRLF. Without normalising, every fragment would read as
|
||||
edited the moment the page was saved once."""
|
||||
default = prompts.catalogue(db)["context.memories"].default
|
||||
prompts.save(db, {"context.memories": default.replace("\n", "\r\n")})
|
||||
assert prompts.stored(db) == {}
|
||||
|
||||
|
||||
def test_restoring_the_default_text_removes_the_override(db):
|
||||
default = prompts.catalogue(db)["core.heading"].default
|
||||
prompts.save(db, {"core.heading": "## One"})
|
||||
prompts.save(db, {"core.heading": default})
|
||||
assert prompts.stored(db) == {}
|
||||
|
||||
|
||||
def test_a_fragment_that_was_not_submitted_keeps_its_override(db):
|
||||
"""A fragment can be missing from the page because whatever contributes it
|
||||
is switched off -- a disabled custom tool. A save must not throw its wording
|
||||
away just for not having been on screen."""
|
||||
prompts.save(db, {"core.heading": "## One", "core.style": "Two"})
|
||||
prompts.save(db, {"core.heading": "## One"})
|
||||
assert set(prompts.stored(db)) == {"core.heading", "core.style"}
|
||||
|
||||
|
||||
def test_plain_settings_in_the_group_survive_a_save(db):
|
||||
settings_store.update(db, {"max_harness_chars": 500}, key=settings_store.PROMPTS)
|
||||
prompts.save(db, {"core.heading": "## One"})
|
||||
assert settings_store.get(db, "max_harness_chars", key=settings_store.PROMPTS) == 500
|
||||
|
||||
|
||||
def test_clear_returns_every_fragment_to_its_default(db):
|
||||
prompts.save(db, {"core.heading": "## One"})
|
||||
prompts.clear(db)
|
||||
assert prompts.stored(db) == {}
|
||||
assert prompts.resolve(db, "core.heading") == "## How to work"
|
||||
|
||||
|
||||
def test_overrides_can_be_supplied_without_touching_the_database(db):
|
||||
"""What the admin page previews unsaved text with."""
|
||||
text = _assembled(db, variables={}, overrides={"core.heading": "## Draft"})
|
||||
assert "## Draft" in text
|
||||
assert prompts.stored(db) == {}
|
||||
|
||||
|
||||
# --- The custom-tool seam ----------------------------------------------------
|
||||
def test_a_registered_source_needs_no_change_anywhere_else(db, extra_source):
|
||||
"""The one requirement the whole design exists for: when custom tools land,
|
||||
a tool contributes its guidance by registering a source and nothing else."""
|
||||
extra_source(
|
||||
prompts.Fragment(
|
||||
key="tool.zz_weather",
|
||||
label="Weather",
|
||||
group=prompts.GROUP_TOOLS,
|
||||
order=500,
|
||||
default="- Check the forecast before answering about weather.",
|
||||
families=("zz_weather",),
|
||||
)
|
||||
)
|
||||
assert "tool.zz_weather" in prompts.catalogue(db)
|
||||
assert "Check the forecast" in _assembled(db, variables={}, families=("zz_weather",))
|
||||
assert any(
|
||||
fragment.key == "tool.zz_weather"
|
||||
for _, _, fragments in prompts.grouped(db)
|
||||
for fragment in fragments
|
||||
)
|
||||
# And it is editable through the same one write path as a built-in.
|
||||
prompts.save(db, {"tool.zz_weather": "- Ask the sky."})
|
||||
assert prompts.resolve(db, "tool.zz_weather") == "- Ask the sky."
|
||||
|
||||
|
||||
def test_the_first_source_to_claim_a_key_keeps_it(db, extra_source):
|
||||
extra_source(
|
||||
prompts.Fragment(
|
||||
key="core.heading", label="x", group=prompts.GROUP_CORE, default="## Hijacked"
|
||||
)
|
||||
)
|
||||
assert prompts.catalogue(db)["core.heading"].default == "## How to work"
|
||||
|
||||
|
||||
# --- The catalogue itself ----------------------------------------------------
|
||||
def test_every_builtin_key_is_unique_and_well_formed():
|
||||
keys = [fragment.key for fragment in prompts.BUILTIN]
|
||||
assert len(keys) == len(set(keys))
|
||||
for key in keys:
|
||||
assert prompts.KEY_PATTERN.match(key), key
|
||||
|
||||
|
||||
def test_every_variable_a_fragment_names_is_documented():
|
||||
"""The legend is the only place a variable is explained, so a fragment
|
||||
referring to one that is not listed is a fragment nobody can use."""
|
||||
for fragment in prompts.BUILTIN:
|
||||
for name in (*fragment.variables, *fragment.requires):
|
||||
assert name in prompts.VARIABLE_NAMES, f"{fragment.key} names {name}"
|
||||
for name in prompts.variables_in(fragment.default):
|
||||
assert name in prompts.VARIABLE_NAMES, f"{fragment.key} uses {name}"
|
||||
|
||||
|
||||
def test_every_variable_a_fragment_uses_is_declared():
|
||||
"""Otherwise the field's own legend chips would not mention it."""
|
||||
for fragment in prompts.BUILTIN:
|
||||
for name in prompts.variables_in(fragment.default):
|
||||
assert name in fragment.variables, f"{fragment.key} uses undeclared {name}"
|
||||
Reference in New Issue
Block a user