diff --git a/src/lembas/api/admin.py b/src/lembas/api/admin.py index f066bea..12055b1 100644 --- a/src/lembas/api/admin.py +++ b/src/lembas/api/admin.py @@ -58,6 +58,7 @@ async def save_general( instance_name: str = Form("LLeMbas"), allow_signup: bool = Form(False), system_prompt: str = Form(""), + compact_threshold: int = Form(95), ) -> Response: """Save instance settings. @@ -70,6 +71,12 @@ async def save_general( "instance_name": instance_name.strip()[:120] or "LLeMbas", "allow_signup": allow_signup, "system_prompt": system_prompt.strip()[:8000], + # 0 is "never"; anything else is clamped into a band where it can + # do some good. 100 is useless -- you cannot compact after + # overflowing -- and below 50 it fires while there is plenty left. + "compact_threshold": ( + 0 if compact_threshold <= 0 else min(max(compact_threshold, 50), 99) + ), }, ) log.info("registration %s by %s", "opened" if allow_signup else "closed", user.email) diff --git a/src/lembas/api/chats.py b/src/lembas/api/chats.py index d349d06..199d977 100644 --- a/src/lembas/api/chats.py +++ b/src/lembas/api/chats.py @@ -19,9 +19,11 @@ from lembas.db.session import session_scope from lembas.security import permissions from lembas.services import audio as audio_service from lembas.services import chat as chat_service +from lembas.services import compaction as compaction_service from lembas.services import files as files_service from lembas.services import generation as generation_service from lembas.services import metrics as metrics_service +from lembas.services import prompts as prompts_service from lembas.services import sse from lembas.services import tools as tools_service from lembas.services.markdown import escape_text, render_markdown @@ -202,6 +204,64 @@ def _pretty(payload: dict) -> str: return text +@router.post("/{chat_id}/compact") +async def compact_chat(request: Request, db: Db, user: RequiredUser, chat_id: str) -> Response: + """Summarise the earlier turns and stop sending them. + + No permission of its own: compaction changes only what one chat sends + upstream, and gating it would mean answering "why can this user not tidy + their own conversation". + """ + chat = _owned_chat(db, chat_id, user.id) + + unfinished = db.scalar( + select(Message).where(Message.chat_id == chat.id, Message.complete.is_(False)) + ) + if unfinished is not None: + # Summarising a transcript that is still being written races + # build_request. Queuing it is a state machine nobody asked for. + raise HTTPException( + status.HTTP_409_CONFLICT, "Wait for the current reply to finish, then compact." + ) + + template = prompts_service.resolve(db, "task.compact") + if not template.strip(): + raise HTTPException( + status.HTTP_409_CONFLICT, + "Compaction is turned off: its prompt is empty under Admin → Prompts.", + ) + + upto = compaction_service.last_complete(db, chat) + if upto is None: + raise HTTPException( + status.HTTP_409_CONFLICT, "There is nothing here to summarise yet." + ) + + endpoint, model_id = chat_service.resolve_endpoint(db, chat) + transcript = compaction_service.transcript(db, chat, upto=upto) + previous = compaction_service.previous_summary_block(chat) + + summary = await chat_service.summarise_for_compaction( + endpoint, + model_id, + transcript=transcript, + previous_summary=previous, + template=template, + ) + if not summary: + raise HTTPException( + status.HTTP_409_CONFLICT, "The model returned no summary, so nothing changed." + ) + + compaction_service.apply(chat, summary=summary, upto=upto) + db.commit() + log.info("chat %s compacted through %s", chat.id, upto.id) + + return templates.TemplateResponse( + request, "chat/_thread.html", {"request": request, **_thread_context(db, chat, user)} + ) + + @router.post("/{chat_id}/keep") async def keep_chat(db: Db, user: RequiredUser, chat_id: str) -> Response: """Stop a temporary chat being temporary. @@ -380,6 +440,7 @@ async def _follow(chat_id: str, message_id: str) -> AsyncIterator[str]: if generation.content: yield sse.event("render", render_markdown(generation.text)) yield sse.event("metrics", _metrics_html(generation)) + yield sse.event("status", escape_text(generation.status)) last_frame = time.monotonic() if generation.done: @@ -451,16 +512,18 @@ def _metrics_html(generation) -> str: def _thread_context(db: DBSession, chat: Chat, user: User) -> dict: """Everything chat/_thread.html needs to render the conversation.""" - messages = list( + everything = list( db.scalars(select(Message).where(Message.chat_id == chat.id).order_by(Message.created_at)) ) + compacted, messages = compaction_service.split(db, chat, everything) return { "chat": chat, "user": user, "messages": messages, + "compacted": compacted, "bodies": { m.id: render_markdown(m.content) - for m in messages + for m in everything if m.role == ROLE_ASSISTANT and m.content }, "models_by_id": {m.model_id: m for m in chat_service.available_models(db, user)}, @@ -556,6 +619,14 @@ async def edit_message( discarded = _messages_after(db, message) for later in discarded: db.delete(later) + + # A rewind to at or before the compaction boundary leaves that boundary + # describing turns that no longer exist. There is no foreign key to null it + # out on an upgraded database, so it is cleared here. + cutoff = compaction_service.cutoff_message(db, chat) + if cutoff is None or compaction_service.moment(message) <= compaction_service.moment(cutoff): + compaction_service.reset(chat) + db.commit() assistant = chat_service.create_message( diff --git a/src/lembas/api/pages.py b/src/lembas/api/pages.py index d26350d..749199c 100644 --- a/src/lembas/api/pages.py +++ b/src/lembas/api/pages.py @@ -12,6 +12,7 @@ from lembas.db.models import Chat, Folder, KnowledgeBase, Message, User from lembas.security import permissions from lembas.services import audio as audio_service from lembas.services import chat as chat_service +from lembas.services import compaction as compaction_service from lembas.services import settings_store from lembas.services import suggestions as suggestions_service from lembas.services.library import documents as documents_service @@ -220,18 +221,21 @@ async def chat_detail(request: Request, db: Db, user: RequiredUser, chat_id: str chat.unread_notified = False db.commit() - messages = list( + everything = list( db.scalars( select(Message).where(Message.chat_id == chat.id).order_by(Message.created_at) ) ) + # Summarised turns are kept and still rendered, behind a divider -- they + # have only stopped being part of the request. + compacted, messages = compaction_service.split(db, chat, everything) # Markdown is rendered once here rather than in the template so the same # helper produces the page and the streamed final frame -- one code path, # no chance of the two disagreeing. bodies = { message.id: render_markdown(message.content) - for message in messages + for message in everything if message.role == "assistant" and message.content } @@ -255,6 +259,7 @@ async def chat_detail(request: Request, db: Db, user: RequiredUser, chat_id: str { "chat": chat, "messages": messages, + "compacted": compacted, "bodies": bodies, "inherited_prompt": inherited, "inherited_from": inherited_from, diff --git a/src/lembas/db/models/chat.py b/src/lembas/db/models/chat.py index 0eedd64..091b472 100644 --- a/src/lembas/db/models/chat.py +++ b/src/lembas/db/models/chat.py @@ -2,9 +2,10 @@ from __future__ import annotations +from datetime import datetime from typing import TYPE_CHECKING, Any -from sqlalchemy import Boolean, ForeignKey, Integer, String, Text +from sqlalchemy import Boolean, DateTime, ForeignKey, Integer, String, Text from sqlalchemy.orm import Mapped, mapped_column, relationship from lembas.db.base import Base, Timestamps, UUIDPrimaryKey @@ -108,6 +109,19 @@ class Chat(UUIDPrimaryKey, Timestamps, Base): unread: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) unread_notified: Mapped[bool] = mapped_column(Boolean, default=False, nullable=False) + # --- Compaction ---------------------------------------------------------- + # A summary of the turns up to `compacted_through_id`, sent in their place. + # The messages themselves are kept and still shown; they simply stop being + # part of the request. See services/compaction.py. + compact_summary: Mapped[str] = mapped_column(Text, default="") + # A plain id, deliberately not a ForeignKey: db/migrations.py compiles only + # the column type, so a REFERENCES clause would exist on a freshly created + # database and not on an upgraded one, and a constraint half the fleet has + # is worse than none. It is validated on every read instead -- the same + # reasoning `model_id` above carries. + compacted_through_id: Mapped[str | None] = mapped_column(String(32)) + compacted_at: Mapped[datetime | None] = mapped_column(DateTime(timezone=True)) + folder: Mapped[Folder | None] = relationship(back_populates="chats") messages: Mapped[list[Message]] = relationship( back_populates="chat", diff --git a/src/lembas/services/chat.py b/src/lembas/services/chat.py index 956eded..2373b12 100644 --- a/src/lembas/services/chat.py +++ b/src/lembas/services/chat.py @@ -174,11 +174,31 @@ def build_messages( resolved, which is how the harness gets in front of the authored prompt without this function knowing anything about tools. """ + from lembas.services import compaction as compaction_service + from lembas.services import prompts as prompts_service + payload: list[dict[str, Any]] = [] system = effective_system_prompt(db, chat) if system_prompt is None else system_prompt if system: payload.append({"role": ROLE_SYSTEM, "content": system}) + # Compacted turns are replaced by a summary carried in two turns rather than + # one. A leading `assistant` breaks templates that require the first + # non-system message to be `user`; a lone leading `user` produces user, user + # whenever the kept history starts on a user turn -- which it always does, + # because the cutoff lands on a finished reply. The pair alternates + # correctly in both directions and keeps exactly one system message. + cutoff = compaction_service.cutoff_message(db, chat) + if cutoff is not None: + lead = prompts_service.resolve(db, "task.compact_lead").strip() + ack = prompts_service.resolve(db, "task.compact_ack").strip() + summary = chat.compact_summary.strip() + payload.append( + {"role": ROLE_USER, "content": f"{lead}\n\n{summary}" if lead else summary} + ) + if ack: + payload.append({"role": ROLE_ASSISTANT, "content": ack}) + history = db.scalars( select(Message).where(Message.chat_id == chat.id).order_by(Message.created_at) ).all() @@ -186,6 +206,10 @@ def build_messages( for message in history: if upto is not None and message.id == upto.id: break + if cutoff is not None and compaction_service.moment( + message + ) <= compaction_service.moment(cutoff): + continue # Skip turns that failed or produced nothing -- but a message carrying # only an attachment has no text and must still be sent. if message.error: @@ -391,6 +415,42 @@ def create_message( return message +async def summarise_for_compaction( + endpoint: Endpoint, + model_id: str, + *, + transcript: str, + previous_summary: str, + template: str, +) -> str: + """Ask the model to summarise the earlier turns. + + `template` is passed in for the same reason `generate_title`'s is: this runs + after the generation's session has closed, and opening another one there is + how you get a session that outlives its scope. An empty template means an + administrator cleared the fragment, and nothing is asked of anyone. + """ + from lembas.services import prompts as prompts_service + + if not template.strip() or not transcript.strip(): + return "" + + prompt = prompts_service.substitute( + template, {"transcript": transcript, "previous_summary": previous_summary} + ) + raw = await complete( + endpoint, + { + "model": model_id, + "messages": [{"role": ROLE_USER, "content": prompt}], + "max_tokens": 1200, + # Low, but not zero: this is recall, not invention. + "temperature": 0.3, + }, + ) + return raw.strip() + + def sweep_temporary(db: DBSession, older_than: timedelta = TEMPORARY_LIFETIME) -> int: """Delete temporary chats nobody has touched for a day. diff --git a/src/lembas/services/compaction.py b/src/lembas/services/compaction.py new file mode 100644 index 0000000..a967ba9 --- /dev/null +++ b/src/lembas/services/compaction.py @@ -0,0 +1,192 @@ +"""Carrying a long conversation forward without carrying all of it. + +Past a certain length every chat stops working: the window fills, and the only +options are to lose the beginning or to start again. Compaction summarises the +earlier turns and sends the summary in their place. + +**The messages are kept.** They stay in the transcript, collapsed behind a +divider, and simply stop being part of the request. A summary that turned out +badly is then a bad turn rather than a lost conversation, which is what makes +the button safe to press and automatic compaction safe to have at all. + +**Stored on the Chat, not as a synthetic Message.** A synthetic row would need a +role: `system` breaks the one-system-message rule the moment `build_messages` +emits it beside the harness, and `user`/`assistant` makes it a turn people can +edit, regenerate from and copy, indistinguishable from a real one in all four +places a bubble is rendered. Worse, "editing rewinds, it does not branch" would +silently delete it and leave no marker that compaction had ever happened. +""" + +from __future__ import annotations + +import logging +from datetime import UTC, datetime + +from sqlalchemy import select +from sqlalchemy.orm import Session as DBSession + +from lembas.db.models import ROLE_ASSISTANT, Chat, Message +from lembas.services import metrics as metrics_service +from lembas.services import settings_store, tokens + +log = logging.getLogger(__name__) + +# What the summariser is shown. Past this the oldest turns are dropped with a +# marker: a transcript that does not fit the window it is protecting is no use. +MAX_TRANSCRIPT_CHARS = 24_000 + +# Settings key, in the GENERAL group. 0 turns automatic compaction off; the +# button still works, because a person asking for it does not need a threshold. +THRESHOLD_KEY = "compact_threshold" +DEFAULT_THRESHOLD = 95 + + +def threshold(db: DBSession) -> int: + value = settings_store.get(db, THRESHOLD_KEY) + return int(value) if isinstance(value, (int, float)) else DEFAULT_THRESHOLD + + +def moment(message: Message) -> datetime: + """A message's timestamp, always comparable. + + SQLite does not store the offset, so a row loaded from disk comes back naive + while one still in the session's identity map keeps the tzinfo it was + created with. Comparing the two raises, and every comparison here is between + exactly those: a cutoff fetched by id against history loaded in bulk. + `files.sweep_orphans` already normalises for the same reason. + """ + created = message.created_at + return created if created.tzinfo is not None else created.replace(tzinfo=UTC) + + +def cutoff_message(db: DBSession, chat: Chat) -> Message | None: + """The message compaction reached, or None if it never has. + + There is no foreign key to null this out on an upgraded database, so the + check is load-bearing rather than defensive: an id pointing at a message + that has been deleted means the boundary no longer describes anything, and + the chat has to read as uncompacted. + """ + if not chat.compact_summary or not chat.compacted_through_id: + return None + message = db.get(Message, chat.compacted_through_id) + if message is None or message.chat_id != chat.id: + return None + return message + + +def reset(chat: Chat) -> None: + """Forget that this chat was ever compacted.""" + chat.compact_summary = "" + chat.compacted_through_id = None + chat.compacted_at = None + + +def apply(chat: Chat, *, summary: str, upto: Message) -> None: + """Record a summary and move the boundary. Caller commits.""" + chat.compact_summary = summary.strip() + chat.compacted_through_id = upto.id + chat.compacted_at = datetime.now(UTC) + + +def split( + db: DBSession, chat: Chat, messages: list[Message] +) -> tuple[list[Message], list[Message]]: + """(summarised, live) -- what is behind the divider, and what is not.""" + cutoff = cutoff_message(db, chat) + if cutoff is None: + return [], list(messages) + boundary = moment(cutoff) + return ( + [m for m in messages if moment(m) <= boundary], + [m for m in messages if moment(m) > boundary], + ) + + +def last_complete(db: DBSession, chat: Chat) -> Message | None: + """The newest finished assistant turn: where compaction should stop. + + Landing on a reply rather than a question means the kept history starts on a + user turn, which is what every chat template expects. + """ + return db.scalar( + select(Message) + .where( + Message.chat_id == chat.id, + Message.role == ROLE_ASSISTANT, + Message.complete.is_(True), + ) + .order_by(Message.created_at.desc()) + ) + + +def transcript(db: DBSession, chat: Chat, *, upto: Message) -> str: + """The turns to summarise, oldest first, as plain text. + + Only the delta since the last compaction: the previous summary is supplied + separately, and the instruction asks for one record, so each summary + subsumes the one before it. Re-summarising the whole chat every time grows + quadratically and eventually exceeds the very window this protects. + """ + previous = cutoff_message(db, chat) + query = select(Message).where( + Message.chat_id == chat.id, + Message.created_at <= upto.created_at, + Message.error == "", + ) + if previous is not None: + query = query.where(Message.created_at > previous.created_at) + + lines: list[str] = [] + for message in db.scalars(query.order_by(Message.created_at)): + body = message.content.strip() + if not body: + continue + lines.append(f"{message.role}: {body}") + + text = "\n\n".join(lines) + if len(text) > MAX_TRANSCRIPT_CHARS: + # Keep the most recent part: the older it is, the more likely the + # previous summary already covers it. + text = "[earlier turns omitted]\n\n" + text[-MAX_TRANSCRIPT_CHARS:] + return text + + +def previous_summary_block(chat: Chat) -> str: + """The earlier summary, headed, or "" on a first compaction. + + Empty is fine to pass straight through: `prompts.substitute` drops a line + that held a known variable and expanded to nothing, so the prompt does not + end up with a hole where a heading was. + """ + if not chat.compact_summary.strip(): + return "" + return "## Summary of even earlier turns\n\n" + chat.compact_summary.strip() + + +def should_compact(db: DBSession, chat: Chat, *, pending: str = "") -> bool: + """Whether the next request should be summarised first. + + Judged from the last reply's recorded usage plus an estimate of the new + turn. True prompt_tokens are only knowable after a response, so a + retrospective figure is the honest basis -- but on its own it is one turn + stale, and fifty thousand characters pasted into the composer would overflow + a window that measured 90% last time. The estimator covers only that delta. + + Never fires when the model's context length is unknown. Acting on a number + nobody supplied is exactly what the 0-means-unknown rule exists to prevent. + """ + limit = threshold(db) + if limit <= 0: + return False + + last = last_complete(db, chat) + if last is None: + return False + + usage = metrics_service.from_message(last.usage_json) + if usage.context_limit <= 0 or usage.context_tokens <= 0: + return False + + projected = usage.context_tokens + tokens.estimate(pending) + return projected >= usage.context_limit * limit / 100 diff --git a/src/lembas/services/generation.py b/src/lembas/services/generation.py index 2e4d9dc..88e9572 100644 --- a/src/lembas/services/generation.py +++ b/src/lembas/services/generation.py @@ -22,9 +22,12 @@ import time from dataclasses import dataclass, field from datetime import UTC, datetime, timedelta +from sqlalchemy import select + from lembas.db.models import ROLE_ASSISTANT, ROLE_USER, Chat, Message, User from lembas.db.session import session_scope from lembas.services import chat as chat_service +from lembas.services import compaction as compaction_service from lembas.services import metrics as metrics_service from lembas.services import prompts as prompts_service from lembas.services import tokens @@ -95,6 +98,10 @@ class Generation: # woken individually: with a 100ms cadence a short poll is simpler than # future bookkeeping, and cannot drop a wakeup. version: int = 0 + # What the reply is doing when it is not producing tokens. Shown in the + # streaming bubble, because a silent multi-second pause before the first + # token is what a hang looks like. + status: str = "" # Number of browsers currently watching. Decides whether a finished reply # counts as unread. followers: int = 0 @@ -214,6 +221,13 @@ async def _run(generation: Generation) -> None: title_prompt = "" try: + # Before the request is assembled, so build_request is called once and + # what goes out is the compacted conversation -- there is no second + # assembly path. Here rather than in post_message because that route's + # whole contract is to return immediately, and a three-second + # summarisation in front of it would break exactly that. + await _maybe_compact(generation) + with session_scope() as db: chat = db.get(Chat, generation.chat_id) message = db.get(Message, generation.message_id) @@ -387,6 +401,77 @@ async def _run(generation: Generation) -> None: generation.touch() +async def _maybe_compact(generation: Generation) -> None: + """Summarise the earlier turns if the window is about to be full. + + Never raises. A failed compaction logs and sends the uncompacted request, + which either works or fails upstream with a message that says what actually + happened -- refusing to answer because the summariser was unavailable would + be a worse trade. + + The awaited call is deliberately outside any session, the same shape titling + uses: read everything needed, close, ask, reopen to write. + """ + try: + with session_scope() as db: + chat = db.get(Chat, generation.chat_id) + message = db.get(Message, generation.message_id) + if chat is None or message is None: + return + + pending = _pending_text(db, message) + if not compaction_service.should_compact(db, chat, pending=pending): + return + + template = prompts_service.resolve(db, "task.compact") + upto = compaction_service.last_complete(db, chat) + if not template.strip() or upto is None: + return + + endpoint, model_id = chat_service.resolve_endpoint(db, chat) + transcript = compaction_service.transcript(db, chat, upto=upto) + previous = compaction_service.previous_summary_block(chat) + upto_id = upto.id + + generation.status = "Summarising earlier messages…" + generation.touch() + + summary = await chat_service.summarise_for_compaction( + endpoint, + model_id, + transcript=transcript, + previous_summary=previous, + template=template, + ) + if not summary: + return + + with session_scope() as db: + chat = db.get(Chat, generation.chat_id) + upto = db.get(Message, upto_id) + if chat is None or upto is None: + return + compaction_service.apply(chat, summary=summary, upto=upto) + db.commit() + log.info("chat %s compacted automatically through %s", chat.id, upto_id) + except Exception: # noqa: BLE001 - the reply matters more than the tidy-up + log.exception("automatic compaction failed for chat %s", generation.chat_id) + finally: + generation.status = "" + generation.touch() + + +def _pending_text(db, message: Message) -> str: + """The user turn this reply is answering, for the size estimate.""" + previous = db.scalars( + select(Message) + .where(Message.chat_id == message.chat_id, Message.created_at < message.created_at) + .order_by(Message.created_at.desc()) + .limit(1) + ).first() + return previous.content if previous is not None else "" + + def _question_from(payload: dict) -> str: """The last thing the user said, for auto-titling.""" for entry in reversed(payload.get("messages", [])): diff --git a/src/lembas/services/prompts.py b/src/lembas/services/prompts.py index 55efa99..e153943 100644 --- a/src/lembas/services/prompts.py +++ b/src/lembas/services/prompts.py @@ -156,6 +156,16 @@ VARIABLES: tuple[Variable, ...] = ( ), Variable("question", "Question", "The first message. Chat title task only."), Variable("answer", "Answer", "The first reply. Chat title task only."), + Variable( + "transcript", + "Transcript", + "The turns being summarised, oldest first. Compaction task only.", + ), + Variable( + "previous_summary", + "Earlier summary", + "The summary from a previous compaction, if there was one. Compaction task only.", + ), ) VARIABLE_NAMES = frozenset(variable.name for variable in VARIABLES) @@ -764,6 +774,77 @@ BUILTIN: tuple[Fragment, ...] = ( "Assistant: {{answer}}" ), ), + Fragment( + key="task.compact", + label="Compaction summary", + group=GROUP_TASKS, + order=410, + variables=("transcript", "previous_summary"), + hint="A separate one-message request, not part of any chat. Clear it to " + "turn compaction off entirely: the button says so and nothing is " + "summarised automatically.", + default=( + "Summarise the conversation below so it can be carried forward after the " + "earlier turns are dropped from your context. This is a working record, " + "not a report for a reader.\n" + "\n" + "Keep, under these headings and in this order:\n" + "\n" + "## What we are doing\n" + "The goal, and where we have got to.\n" + "\n" + "## Decisions\n" + "Anything settled, and why. A decision without its reason gets argued " + "again.\n" + "\n" + "## Facts established\n" + "Names, numbers, versions, file paths, URLs and identifiers, copied " + "exactly. Do not round them, paraphrase them or reconstruct one from " + "memory — if it is not in the transcript, leave it out.\n" + "\n" + "## Open threads\n" + "What is unfinished, and what was about to happen next.\n" + "\n" + "Leave out pleasantries, retracted ideas and anything already superseded. " + "Do not answer the conversation: you are recording it. Write in the " + "language of the conversation, and stay under 500 words.\n" + "\n" + "{{previous_summary}}\n" + "\n" + "## Transcript\n" + "\n" + "{{transcript}}" + ), + ), + Fragment( + key="task.compact_lead", + label="How a summary is introduced", + group=GROUP_TASKS, + order=420, + hint="Sits in front of the summary, in the turn that replaces the " + "messages no longer being sent. Without it a model reads the summary as " + "something the person has just typed.", + default=( + "Here is a summary of the earlier part of this conversation. Those " + "messages are no longer in your context. Treat this summary as an " + "accurate record of them and rely on it rather than on what you can no " + "longer see; if it does not cover something you need, say so instead of " + "filling the gap." + ), + ), + Fragment( + key="task.compact_ack", + label="The model's acknowledgement", + group=GROUP_TASKS, + order=430, + hint="One assistant turn after the summary, so the conversation still " + "alternates user, assistant, user. Several chat templates reject a " + "history that does not.", + default=( + "Understood. I have the summary of the earlier turns and will carry on " + "from there." + ), + ), ) register_source(_builtin_source) diff --git a/src/lembas/services/settings_store.py b/src/lembas/services/settings_store.py index b4c101a..ec9caa4 100644 --- a/src/lembas/services/settings_store.py +++ b/src/lembas/services/settings_store.py @@ -35,6 +35,12 @@ def _general_defaults() -> dict[str, Any]: # Applied to every chat that has no model or chat prompt of its # own. See services.chat.effective_system_prompt. "system_prompt": "", + # Percentage of a model's context length at which the earlier turns are + # summarised automatically. 0 turns it off; the Compact button still + # works, because a person asking for it does not need a threshold. + # Never fires for a model whose context_length is 0, since that is + # "unknown" rather than "small". See services/compaction.py. + "compact_threshold": 95, } diff --git a/src/lembas/web/static/css/chat.css b/src/lembas/web/static/css/chat.css index a63a94b..a907820 100644 --- a/src/lembas/web/static/css/chat.css +++ b/src/lembas/web/static/css/chat.css @@ -167,6 +167,45 @@ .reasoning__summary::-webkit-details-marker { display: none; } .reasoning__summary:hover { color: var(--ink); background: var(--surface-hover); } +/* --- Compacted turns ------------------------------------------------------- + Summarised messages, kept and readable but out of the way. Collapsed by + default: the point of compacting was that they stopped mattering. +*/ +.compacted { + border: 1px dashed var(--border-strong); + border-radius: var(--radius); + background: color-mix(in srgb, var(--surface) 50%, transparent); +} +.compacted__summary { + display: flex; + align-items: center; + gap: var(--sp-2); + padding: var(--sp-2) var(--sp-3); + cursor: pointer; + color: var(--ink-muted); + font-size: var(--text-sm); + list-style: none; + user-select: none; +} +.compacted__summary::-webkit-details-marker { display: none; } +.compacted__summary:hover { color: var(--ink); } +.compacted[open] .reasoning__chevron { transform: rotate(180deg); } + +.compacted__note { + margin: 0; + padding: 0 var(--sp-3); + font-size: var(--text-xs); + color: var(--ink-faint); + line-height: var(--leading-normal); +} +.compacted__body { + display: flex; + flex-direction: column; + gap: var(--sp-6); + padding: var(--sp-4) var(--sp-3); + opacity: 0.75; +} + /* --- Suggestions ----------------------------------------------------------- Starting points on the empty screen. Cards rather than a list, because they are things to press. @@ -333,6 +372,9 @@ } /* --- Stop, notes and editing ---------------------------------------------- */ +.msg__status { font-size: var(--text-xs); color: var(--ink-faint); font-style: italic; } +.msg__status:empty { display: none; } + .msg__waiting { display: flex; align-items: center; diff --git a/src/lembas/web/templates/admin/general.html b/src/lembas/web/templates/admin/general.html index 83759b3..0339701 100644 --- a/src/lembas/web/templates/admin/general.html +++ b/src/lembas/web/templates/admin/general.html @@ -41,6 +41,29 @@ +
+

Compaction

+

+ A long conversation eventually fills the model's context. When it gets + close, the earlier turns are summarised and the summary is sent in their + place. The messages themselves are kept and stay readable in the + transcript — they simply stop being sent. +

+
+ + +

+ Percent of the model's context length. 0 turns automatic + compaction off; the button in each chat still works. Nothing happens for + a model whose context length is unset under + Models — that is "unknown", not "small", and + this will not act on a number nobody supplied. The wording of the + summary is under Prompts. +

+
+
+

Registration diff --git a/src/lembas/web/templates/chat/_message.html b/src/lembas/web/templates/chat/_message.html index 83756df..4cb906d 100644 --- a/src/lembas/web/templates/chat/_message.html +++ b/src/lembas/web/templates/chat/_message.html @@ -119,6 +119,10 @@ reply is being written, which is where the hand already is. #}
+ {# What the reply is doing when it is not producing tokens. A silent + multi-second pause before the first token is what a hang looks + like. #} +
{# Counts as the reply is written. Everything is an estimate until the usage chunk lands at the very end, and the chips say so. #} diff --git a/src/lembas/web/templates/chat/_thread.html b/src/lembas/web/templates/chat/_thread.html index c3c1afa..afcad66 100644 --- a/src/lembas/web/templates/chat/_thread.html +++ b/src/lembas/web/templates/chat/_thread.html @@ -1,12 +1,38 @@ +{% from "_macros.html" import icon %} {# - The whole thread. Returned after a rewind, which changes an arbitrary number - of messages at once -- replacing the lot is simpler and less error-prone than - working out which individual bubbles to remove. Also included by - chat/index.html, so the conversation is described in exactly one place. + The whole thread. Returned after a rewind or a compaction, either of which + changes an arbitrary number of messages at once -- replacing the lot is + simpler and less error-prone than working out which individual bubbles to + remove. Also included by chat/index.html, so the conversation is described in + exactly one place. Deliberately has no root element: it is swapped with innerHTML into #thread, and an outerHTML swap would take the container with it. #} +{% if compacted %} +{# Summarised turns are kept and still readable -- they have only stopped being + sent. A compaction that summarised badly is then a bad turn rather than a + lost conversation. #} +
+ + {{ icon("archive", "icon--sm") }} + {{ compacted | length }} earlier messages, summarised + {{ icon("chevron-down", "icon--sm reasoning__chevron") }} + +

+ These are no longer sent to the model; a summary of them goes instead. They + are kept here so nothing is lost. +

+
+ {% for message in compacted %} + {% with body_html = bodies.get(message.id, "") %} + {% include "chat/_message.html" %} + {% endwith %} + {% endfor %} +
+
+{% endif %} + {% for message in messages %} {% with body_html = bodies.get(message.id, "") %} {% include "chat/_message.html" %} diff --git a/src/lembas/web/templates/chat/index.html b/src/lembas/web/templates/chat/index.html index e776516..97f40a1 100644 --- a/src/lembas/web/templates/chat/index.html +++ b/src/lembas/web/templates/chat/index.html @@ -66,6 +66,18 @@ {% endif %} + {% if chat and messages %} + + {% endif %} + {% if chat and user.is_admin %}