Compaction: a button, and automatically when the window fills
A long conversation eventually just stops working. Compaction summarises the earlier turns and sends the summary in their place. The messages are kept. They stay in the transcript behind a collapsed divider and simply stop being part of the request, which is what makes the button safe to press and automatic compaction safe to have at all: a summary that came out badly is a bad turn, not a lost conversation. Stored on the Chat, not as a synthetic Message. A synthetic row needs a role -- `system` breaks the one-system-message rule the moment build_messages emits it beside the harness, and user/assistant makes it a turn people can edit, regenerate from and copy, indistinguishable from a real one in all four places a bubble is rendered. Worse, "editing rewinds, it does not branch" would silently delete it and leave no marker that compaction had happened at all. The summary goes out as a user turn and an assistant turn, not one. A leading assistant breaks templates requiring the first non-system message to be user; a lone leading user produces user, user whenever the kept history starts on a user turn -- which it always does, because the cutoff lands on a finished reply. compacted_through_id is a plain id rather than a foreign key: migrations.py compiles only the column type, so a REFERENCES clause would exist on a fresh database and not on an upgraded one, and a constraint half the fleet has is worse than none. cutoff_message validates it on every read instead, and a rewind past the boundary clears it. Compacting again summarises only the delta, with the previous summary supplied to be subsumed. Re-summarising the whole chat each time grows quadratically and eventually exceeds the window it is protecting. Automatically at the top of _run, not in post_message: that route's contract is to return immediately and leave the slow part to a resumable connection, and it also means build_request is called once, after compaction, with no second assembly path. The trigger is the last reply's recorded usage plus an estimate of the new turn -- retrospective because true prompt_tokens are only knowable after a response, plus the delta because otherwise fifty thousand characters pasted into the composer overflow a window that read 90% last turn. It never fires when the context length is unknown. It does fire on estimated counts, which is safe here precisely because nothing is lost. _maybe_compact never raises: a failure logs and sends the uncompacted request. A `status` event says "Summarising earlier messages…" in the meantime, because a silent multi-second pause before the first token is what a hang looks like. The wording is three fragments under Admin - Prompts. Clearing task.compact turns compaction off entirely. Also adds compaction.moment(): SQLite does not store the offset, so a row loaded from disk is naive while one in the session's identity map keeps its tzinfo, and comparing the two raises. Every comparison here is between exactly those. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
@@ -0,0 +1,372 @@
|
||||
"""Compaction: what is summarised, what is sent, and when it happens by itself."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from datetime import UTC, datetime, timedelta
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
from sqlalchemy import select
|
||||
|
||||
from lembas.db.models import Chat, Connection, Message, Model
|
||||
from lembas.services import chat as chat_service
|
||||
from lembas.services import compaction as compaction_service
|
||||
from lembas.services import prompts, settings_store
|
||||
from lembas.services.crypto import encrypt
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def chat(db, registered, make_chat) -> Chat:
|
||||
connection = Connection(
|
||||
name="Test", base_url="http://x.test", api_key_encrypted=encrypt("")
|
||||
)
|
||||
db.add(connection)
|
||||
db.commit()
|
||||
db.add(
|
||||
Model(connection_id=connection.id, model_id="test-model", context_length=1000)
|
||||
)
|
||||
db.commit()
|
||||
return db.get(Chat, make_chat())
|
||||
|
||||
|
||||
def _exchange(db, chat: Chat, *, question: str, answer: str, minutes: int = 0) -> Message:
|
||||
"""One user turn and its reply, backdated so ordering is deterministic."""
|
||||
when = datetime.now(UTC) - timedelta(minutes=minutes)
|
||||
db.add(Message(chat_id=chat.id, role="user", content=question, created_at=when))
|
||||
reply = Message(
|
||||
chat_id=chat.id,
|
||||
role="assistant",
|
||||
content=answer,
|
||||
created_at=when + timedelta(seconds=1),
|
||||
)
|
||||
db.add(reply)
|
||||
db.commit()
|
||||
return reply
|
||||
|
||||
|
||||
def _usage(reply: Message, *, context_tokens: int, limit: int = 1000) -> None:
|
||||
reply.usage_json = {
|
||||
"prompt_tokens": context_tokens,
|
||||
"completion_tokens": 0,
|
||||
"total_tokens": context_tokens,
|
||||
"context_tokens": context_tokens,
|
||||
"context_limit": limit,
|
||||
"estimated": False,
|
||||
"elapsed_ms": 10,
|
||||
"rounds": 1,
|
||||
}
|
||||
|
||||
|
||||
# --- The boundary -------------------------------------------------------------
|
||||
def test_an_uncompacted_chat_splits_into_nothing_and_everything(db, chat):
|
||||
_exchange(db, chat, question="one", answer="two")
|
||||
messages = list(db.scalars(select(Message).order_by(Message.created_at)))
|
||||
assert compaction_service.split(db, chat, messages) == ([], messages)
|
||||
|
||||
|
||||
def test_a_dangling_cutoff_reads_as_uncompacted(db, chat):
|
||||
"""There is no foreign key to null it out on an upgraded database, so the
|
||||
guard is load-bearing rather than defensive."""
|
||||
reply = _exchange(db, chat, question="one", answer="two", minutes=10)
|
||||
compaction_service.apply(chat, summary="a summary", upto=reply)
|
||||
db.commit()
|
||||
|
||||
db.delete(reply)
|
||||
db.commit()
|
||||
|
||||
assert compaction_service.cutoff_message(db, chat) is None
|
||||
assert compaction_service.split(db, chat, [])[0] == []
|
||||
|
||||
|
||||
def test_the_cutoff_lands_on_a_finished_reply(db, chat):
|
||||
"""So the kept history starts on a user turn, which is what every chat
|
||||
template expects."""
|
||||
_exchange(db, chat, question="one", answer="two", minutes=10)
|
||||
db.add(Message(chat_id=chat.id, role="user", content="three"))
|
||||
db.commit()
|
||||
|
||||
assert compaction_service.last_complete(db, chat).content == "two"
|
||||
|
||||
|
||||
# --- What gets sent -----------------------------------------------------------
|
||||
def test_the_summary_replaces_the_compacted_turns(db, chat):
|
||||
old = _exchange(db, chat, question="what is lembas?", answer="Waybread.", minutes=10)
|
||||
_exchange(db, chat, question="how much?", answer="A bite.", minutes=1)
|
||||
compaction_service.apply(chat, summary="They asked about lembas.", upto=old)
|
||||
db.commit()
|
||||
|
||||
messages = chat_service.build_messages(db, chat)
|
||||
contents = [m["content"] for m in messages]
|
||||
|
||||
assert "what is lembas?" not in contents
|
||||
assert "how much?" in contents
|
||||
assert any("They asked about lembas." in c for c in contents)
|
||||
|
||||
|
||||
def test_the_summary_is_carried_by_a_user_and_an_assistant_turn(db, chat):
|
||||
"""A leading assistant turn breaks templates requiring the first non-system
|
||||
message to be user; a lone leading user turn produces user, user whenever the
|
||||
kept history starts on a user turn -- which it always does."""
|
||||
old = _exchange(db, chat, question="one", answer="two", minutes=10)
|
||||
_exchange(db, chat, question="three", answer="four", minutes=1)
|
||||
compaction_service.apply(chat, summary="Summary.", upto=old)
|
||||
db.commit()
|
||||
|
||||
roles = [m["role"] for m in chat_service.build_messages(db, chat)]
|
||||
assert roles[:2] == ["user", "assistant"]
|
||||
# And it still alternates from there.
|
||||
assert roles == ["user", "assistant", "user", "assistant"]
|
||||
|
||||
|
||||
def test_there_is_still_exactly_one_system_message(db, chat):
|
||||
settings_store.update(db, {"system_prompt": "Speak as Gandalf."})
|
||||
old = _exchange(db, chat, question="one", answer="two", minutes=10)
|
||||
compaction_service.apply(chat, summary="Summary.", upto=old)
|
||||
db.commit()
|
||||
|
||||
roles = [m["role"] for m in chat_service.build_messages(db, chat, system_prompt="S")]
|
||||
assert roles.count("system") == 1
|
||||
assert roles[0] == "system"
|
||||
|
||||
|
||||
def test_clearing_the_lead_fragment_still_sends_the_summary(db, chat):
|
||||
prompts.save(db, {"task.compact_lead": "", "task.compact_ack": ""})
|
||||
old = _exchange(db, chat, question="one", answer="two", minutes=10)
|
||||
compaction_service.apply(chat, summary="Summary.", upto=old)
|
||||
db.commit()
|
||||
|
||||
messages = chat_service.build_messages(db, chat)
|
||||
assert messages[0] == {"role": "user", "content": "Summary."}
|
||||
|
||||
|
||||
# --- The transcript -----------------------------------------------------------
|
||||
def test_only_the_delta_is_summarised_the_second_time(db, chat):
|
||||
"""Re-summarising the whole chat grows quadratically and eventually exceeds
|
||||
the very window it is protecting."""
|
||||
first = _exchange(db, chat, question="the old part", answer="ok", minutes=20)
|
||||
compaction_service.apply(chat, summary="Earlier summary.", upto=first)
|
||||
db.commit()
|
||||
|
||||
second = _exchange(db, chat, question="the new part", answer="ok", minutes=5)
|
||||
transcript = compaction_service.transcript(db, chat, upto=second)
|
||||
|
||||
assert "the new part" in transcript
|
||||
assert "the old part" not in transcript
|
||||
assert "Earlier summary." in compaction_service.previous_summary_block(chat)
|
||||
|
||||
|
||||
def test_a_first_compaction_has_no_previous_summary(db, chat):
|
||||
assert compaction_service.previous_summary_block(chat) == ""
|
||||
|
||||
|
||||
def test_a_huge_transcript_is_trimmed_from_the_front(db, chat):
|
||||
reply = _exchange(db, chat, question="x" * 40_000, answer="ok", minutes=5)
|
||||
transcript = compaction_service.transcript(db, chat, upto=reply)
|
||||
|
||||
assert len(transcript) < 40_000
|
||||
assert transcript.startswith("[earlier turns omitted]")
|
||||
|
||||
|
||||
# --- The button ---------------------------------------------------------------
|
||||
def _summariser(text: str = "## What we are doing\n\nAsking about lembas."):
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
return httpx.Response(200, json={"choices": [{"message": {"content": text}}]})
|
||||
|
||||
return handler
|
||||
|
||||
|
||||
async def test_compacting_summarises_and_hides_the_earlier_turns(
|
||||
client: TestClient, db, chat, mock_http
|
||||
):
|
||||
_exchange(db, chat, question="what is lembas?", answer="Waybread.", minutes=10)
|
||||
mock_http(_summariser())
|
||||
|
||||
response = client.post(f"/api/chats/{chat.id}/compact")
|
||||
assert response.status_code == 200
|
||||
|
||||
db.refresh(chat)
|
||||
assert "Asking about lembas." in chat.compact_summary
|
||||
assert chat.compacted_through_id
|
||||
# The messages are still there, behind the divider.
|
||||
assert "earlier messages, summarised" in response.text
|
||||
assert db.scalar(select(Message).where(Message.content == "what is lembas?")) is not None
|
||||
|
||||
|
||||
async def test_compacting_while_a_reply_is_being_written_is_refused(
|
||||
client: TestClient, db, chat, mock_http
|
||||
):
|
||||
_exchange(db, chat, question="one", answer="two", minutes=10)
|
||||
db.add(Message(chat_id=chat.id, role="assistant", content="", complete=False))
|
||||
db.commit()
|
||||
|
||||
response = client.post(f"/api/chats/{chat.id}/compact")
|
||||
assert response.status_code == 409
|
||||
|
||||
|
||||
async def test_compaction_can_be_turned_off_by_clearing_its_prompt(
|
||||
client: TestClient, db, chat, mock_http
|
||||
):
|
||||
_exchange(db, chat, question="one", answer="two", minutes=10)
|
||||
prompts.save(db, {"task.compact": ""})
|
||||
|
||||
response = client.post(f"/api/chats/{chat.id}/compact")
|
||||
assert response.status_code == 409
|
||||
assert "turned off" in response.json()["detail"]
|
||||
|
||||
|
||||
async def test_compacting_an_empty_chat_is_refused(client: TestClient, db, chat, mock_http):
|
||||
assert client.post(f"/api/chats/{chat.id}/compact").status_code == 409
|
||||
|
||||
|
||||
# --- Rewinding across the boundary --------------------------------------------
|
||||
def test_editing_at_or_before_the_cutoff_clears_the_compaction(
|
||||
client: TestClient, db, chat, mock_http
|
||||
):
|
||||
"""A rewind deletes everything after the edited message, so a boundary at or
|
||||
behind it no longer describes anything that exists."""
|
||||
first_reply = _exchange(db, chat, question="one", answer="two", minutes=20)
|
||||
_exchange(db, chat, question="three", answer="four", minutes=10)
|
||||
compaction_service.apply(chat, summary="Summary.", upto=first_reply)
|
||||
db.commit()
|
||||
|
||||
first_user = db.scalar(select(Message).where(Message.content == "one"))
|
||||
mock_http(_summariser())
|
||||
client.post(
|
||||
f"/api/chats/{chat.id}/messages/{first_user.id}/edit", data={"content": "one again"}
|
||||
)
|
||||
|
||||
db.refresh(chat)
|
||||
assert chat.compact_summary == ""
|
||||
assert chat.compacted_through_id is None
|
||||
|
||||
|
||||
# --- Automatic ----------------------------------------------------------------
|
||||
def test_it_fires_when_the_window_is_nearly_full(db, chat):
|
||||
reply = _exchange(db, chat, question="one", answer="two", minutes=5)
|
||||
_usage(reply, context_tokens=960)
|
||||
db.commit()
|
||||
|
||||
assert compaction_service.should_compact(db, chat) is True
|
||||
|
||||
|
||||
def test_it_does_not_fire_with_room_to_spare(db, chat):
|
||||
reply = _exchange(db, chat, question="one", answer="two", minutes=5)
|
||||
_usage(reply, context_tokens=400)
|
||||
db.commit()
|
||||
|
||||
assert compaction_service.should_compact(db, chat) is False
|
||||
|
||||
|
||||
def test_a_large_pending_turn_is_counted(db, chat):
|
||||
"""The recorded figure is one turn stale. Fifty thousand characters pasted
|
||||
into the composer overflow a window that measured 90% last time."""
|
||||
reply = _exchange(db, chat, question="one", answer="two", minutes=5)
|
||||
_usage(reply, context_tokens=900)
|
||||
db.commit()
|
||||
|
||||
assert compaction_service.should_compact(db, chat) is False
|
||||
assert compaction_service.should_compact(db, chat, pending="x" * 400) is True
|
||||
|
||||
|
||||
def test_it_never_fires_without_a_context_length(db, chat):
|
||||
"""Acting on a number nobody supplied is exactly what 0-means-unknown is
|
||||
there to prevent."""
|
||||
reply = _exchange(db, chat, question="one", answer="two", minutes=5)
|
||||
_usage(reply, context_tokens=99_000, limit=0)
|
||||
db.commit()
|
||||
|
||||
assert compaction_service.should_compact(db, chat) is False
|
||||
|
||||
|
||||
def test_a_threshold_of_zero_turns_it_off(db, chat):
|
||||
settings_store.update(db, {"compact_threshold": 0})
|
||||
reply = _exchange(db, chat, question="one", answer="two", minutes=5)
|
||||
_usage(reply, context_tokens=999)
|
||||
db.commit()
|
||||
|
||||
assert compaction_service.should_compact(db, chat) is False
|
||||
|
||||
|
||||
def test_it_does_not_fire_on_the_first_turn(db, chat):
|
||||
assert compaction_service.should_compact(db, chat) is False
|
||||
|
||||
|
||||
def test_it_acts_on_estimated_counts_too(db, chat):
|
||||
"""A premature compaction costs one turn of answer quality, not data -- the
|
||||
messages are still there. That is what makes acting on an estimate safe."""
|
||||
reply = _exchange(db, chat, question="one", answer="two", minutes=5)
|
||||
_usage(reply, context_tokens=960)
|
||||
reply.usage_json = {**reply.usage_json, "estimated": True}
|
||||
db.commit()
|
||||
|
||||
assert compaction_service.should_compact(db, chat) is True
|
||||
|
||||
|
||||
async def test_a_generation_compacts_before_it_asks(db, chat, mock_http):
|
||||
"""At the top of _run, so build_request is called once and what goes out is
|
||||
the compacted conversation."""
|
||||
from lembas.services import generation as generation_service
|
||||
|
||||
reply = _exchange(db, chat, question="what is lembas?", answer="Waybread.", minutes=10)
|
||||
_usage(reply, context_tokens=980)
|
||||
db.commit()
|
||||
|
||||
pending = Message(chat_id=chat.id, role="user", content="more?")
|
||||
db.add(pending)
|
||||
db.commit()
|
||||
placeholder = Message(chat_id=chat.id, role="assistant", content="", complete=False)
|
||||
db.add(placeholder)
|
||||
db.commit()
|
||||
|
||||
sent: list[dict] = []
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
body = json.loads(request.content)
|
||||
sent.append(body)
|
||||
if body.get("stream"):
|
||||
return httpx.Response(200, text="data: [DONE]\n\n")
|
||||
return httpx.Response(200, json={"choices": [{"message": {"content": "A summary."}}]})
|
||||
|
||||
mock_http(handler)
|
||||
generation = generation_service.Generation(chat_id=chat.id, message_id=placeholder.id)
|
||||
generation_service._RUNNING[placeholder.id] = generation
|
||||
await generation_service._run(generation)
|
||||
generation_service._RUNNING.clear()
|
||||
|
||||
db.expire_all()
|
||||
assert db.get(Chat, chat.id).compact_summary == "A summary."
|
||||
# The streamed request went out after compaction, carrying the summary.
|
||||
streamed = next(b for b in sent if b.get("stream"))
|
||||
assert any("A summary." in str(m.get("content")) for m in streamed["messages"])
|
||||
assert not any(m.get("content") == "what is lembas?" for m in streamed["messages"])
|
||||
|
||||
|
||||
async def test_a_failed_compaction_still_sends_the_reply(db, chat, mock_http):
|
||||
"""Refusing to answer because the summariser was unavailable is a worse
|
||||
trade than sending the uncompacted request."""
|
||||
from lembas.services import generation as generation_service
|
||||
|
||||
reply = _exchange(db, chat, question="one", answer="two", minutes=10)
|
||||
_usage(reply, context_tokens=980)
|
||||
db.commit()
|
||||
placeholder = Message(chat_id=chat.id, role="assistant", content="", complete=False)
|
||||
db.add(placeholder)
|
||||
db.commit()
|
||||
|
||||
def handler(request: httpx.Request) -> httpx.Response:
|
||||
if json.loads(request.content).get("stream"):
|
||||
return httpx.Response(
|
||||
200, text='data: {"choices":[{"delta":{"content":"hi"}}]}\n\ndata: [DONE]\n\n'
|
||||
)
|
||||
return httpx.Response(500, json={"error": {"message": "no"}})
|
||||
|
||||
mock_http(handler)
|
||||
generation = generation_service.Generation(chat_id=chat.id, message_id=placeholder.id)
|
||||
generation_service._RUNNING[placeholder.id] = generation
|
||||
await generation_service._run(generation)
|
||||
generation_service._RUNNING.clear()
|
||||
|
||||
assert generation.text == "hi"
|
||||
assert not generation.error
|
||||
Reference in New Issue
Block a user