"""Pausing a reply to ask the reader something. Driven through the real generation loop with a scripted endpoint, because the things worth pinning here are all about the loop: that Stop still works while nothing is streaming, that a decision lands at the right index, and that the card clears itself. """ from __future__ import annotations import asyncio import pytest from sqlalchemy import select from lembas.db.models import ROLE_ASSISTANT, Chat, Connection, Message, Model from lembas.services import generation as generation_service from lembas.services import interaction, settings_store def _chat_that_can_ask(db, user_id): connection = Connection(name="c", base_url="http://127.0.0.1:1", api_key_encrypted="") db.add(connection) db.commit() db.add(Model(connection_id=connection.id, model_id="m", capabilities_json={"tools": True})) db.commit() chat = Chat(user_id=user_id, model_id="m", connection_id=connection.id) db.add(chat) db.commit() db.add(Message(chat_id=chat.id, role="user", content="Which one?", complete=True)) db.commit() assistant = Message(chat_id=chat.id, role=ROLE_ASSISTANT, content="", complete=False) db.add(assistant) db.commit() return chat.id, assistant.id def _ask_chunk(question: str, options: list[str] | None = None, *, index: int = 0, call_id="c1"): import json as _json arguments = {"question": question} if options is not None: arguments["options"] = options return { "choices": [ { "delta": { "tool_calls": [ { "index": index, "id": call_id, "function": { "name": "ask_user", "arguments": _json.dumps(arguments), }, } ] } } ] } def _text_chunk(text: str) -> dict: return {"choices": [{"delta": {"content": text}}]} def _stub_stream(rounds, seen): async def stream_chat(_endpoint, payload): seen.append(payload) for chunk in rounds[min(len(seen) - 1, len(rounds) - 1)]: yield chunk return stream_chat async def _until_paused(generation, *, timeout: float = 2.0): """Wait for the card to go up.""" deadline = asyncio.get_running_loop().time() + timeout while asyncio.get_running_loop().time() < deadline: if generation.pending is not None: return generation.pending await asyncio.sleep(0.01) raise AssertionError("the reply never paused") @pytest.fixture def scripted(db, user_id, monkeypatch): """A reply that asks one question, then answers with whatever it was told.""" chat_id, message_id = _chat_that_can_ask(db, user_id) payloads: list[dict] = [] monkeypatch.setattr( generation_service, "stream_chat", _stub_stream( [[_ask_chunk("Tea or coffee?", ["Tea", "Coffee"])], [_text_chunk("Right you are.")]], payloads, ), ) generation = generation_service.Generation(chat_id=chat_id, message_id=message_id) return generation, payloads, chat_id, message_id # --- The tool is offered at all ---------------------------------------------- def test_ask_user_is_offered_to_an_ordinary_chat(db, user_id): """Not an agent feature. A model in a plain conversation should be able to stop and ask which of two things you meant.""" from lembas.db.models import User from lembas.services import tools as tools_service chat_id, _message_id = _chat_that_can_ask(db, user_id) chat = db.get(Chat, chat_id) offered = tools_service.resolve_tools(db, chat, db.get(User, user_id)) assert "ask_user" in offered.by_name def test_ask_user_is_withheld_without_the_permission(db, user_id): from lembas.db.models import User from lembas.services import tools as tools_service user = db.get(User, user_id) # Administrators are given every permission, so the baseline only bites a # plain account. user.role = "user" settings_store.update(db, {"default_permissions": {"tools.ask": False}}) db.commit() chat = db.get(Chat, _chat_that_can_ask(db, user_id)[0]) assert "ask_user" not in tools_service.resolve_tools(db, chat, user).by_name # --- The pause --------------------------------------------------------------- async def test_the_reply_pauses_and_the_card_describes_the_question(scripted): generation, _payloads, _chat_id, _message_id = scripted task = asyncio.create_task(generation_service._run(generation)) pending = await _until_paused(generation) assert pending.kind == interaction.KIND_QUESTION assert pending.items[0].title == "Tea or coffee?" assert pending.options == ("Tea", "Coffee") assert "Waiting for your answer" in generation.status pending.resolve(interaction.ANSWER, text="Tea") await task async def test_the_answer_reaches_the_model_as_a_tool_result(scripted): generation, payloads, _chat_id, _message_id = scripted task = asyncio.create_task(generation_service._run(generation)) pending = await _until_paused(generation) pending.resolve(interaction.ANSWER, text="Coffee, please") await task turns = [m for m in payloads[1]["messages"] if m.get("role") == "tool"] assert len(turns) == 1 assert "Coffee, please" in turns[0]["content"] assert turns[0]["tool_call_id"] == "c1" async def test_the_card_is_cleared_once_it_is_answered(scripted): generation, _payloads, _chat_id, _message_id = scripted task = asyncio.create_task(generation_service._run(generation)) pending = await _until_paused(generation) version = generation.version pending.resolve(interaction.ANSWER, text="Tea") await task assert generation.pending is None assert generation.version > version, "clearing has to bump the version or no frame is sent" async def test_the_transcript_keeps_what_was_asked_and_answered(scripted): generation, _payloads, _chat_id, _message_id = scripted task = asyncio.create_task(generation_service._run(generation)) pending = await _until_paused(generation) pending.resolve(interaction.ANSWER, text="Tea") await task event = generation.tool_events[0] assert event["kind"] == "ask" assert event["query"] == "Tea or coffee?" assert event["text"] == "Tea" # --- Stop, while nothing is streaming ---------------------------------------- async def test_stop_ends_a_reply_that_is_waiting_for_an_answer(db, user_id, monkeypatch): """`cancel` is read only between streamed chunks, and there are no chunks while the card is up. Without the wakeup in request_stop the button does nothing at all here.""" chat_id, message_id = _chat_that_can_ask(db, user_id) monkeypatch.setattr( generation_service, "stream_chat", _stub_stream([[_ask_chunk("Tea or coffee?")], [_text_chunk("unreachable")]], []), ) generation = generation_service.ensure(chat_id, message_id) await _until_paused(generation) assert generation_service.request_stop(message_id) is True await asyncio.wait_for(asyncio.shield(generation_service._TASKS[message_id]), timeout=2) assert generation.stopped is True assert generation.pending is None # --- Timeout ------------------------------------------------------------------ async def test_an_unanswered_question_expires_and_the_reply_finishes(db, user_id, monkeypatch): chat_id, message_id = _chat_that_can_ask(db, user_id) payloads: list[dict] = [] monkeypatch.setattr( generation_service, "stream_chat", _stub_stream([[_ask_chunk("Tea or coffee?")], [_text_chunk("Never mind.")]], payloads), ) # The clamp floor is 60s, so the timeout is forced directly rather than # through the settings. monkeypatch.setattr( generation_service.tools_service, "context_for", lambda *a, **k: _fast_context(*a, **k), ) generation = generation_service.Generation(chat_id=chat_id, message_id=message_id) await asyncio.wait_for(generation_service._run(generation), timeout=5) turns = [m for m in payloads[1]["messages"] if m.get("role") == "tool"] assert "did not answer" in turns[0]["content"] assert generation.tool_events[0]["status"] == "error" def _fast_context(db, user, chat=None, *, tools=None): from lembas.services import tools as tools_service context = tools_service.ToolContext( owner_id=user.id if user else "", tools=tools.by_name if tools is not None else None, ) context.interaction_timeout = 0.2 return context # --- The primitive on its own ------------------------------------------------- def test_resolving_twice_only_counts_once(): """Two tabs, or a double click. The second answer must not win.""" async def go(): pause = interaction.build("abc", [_item()], timeout=5) assert pause.resolve(interaction.ANSWER, text="first") is True assert pause.resolve(interaction.ANSWER, text="second") is False assert (await pause._future).text == "first" asyncio.run(go()) def test_an_interruption_with_no_future_cannot_be_resolved(): pause = interaction.Interruption(id="abc", items=(_item(),)) assert pause.resolve(interaction.ANSWER) is False def _item() -> interaction.Item: return interaction.Item( index=0, kind=interaction.KIND_QUESTION, tool_name="ask_user", title="Tea or coffee?", ) # --- Resolving one ------------------------------------------------------------- # `answer()` is exercised in-process rather than over the TestClient, because a # future belongs to the loop that made it and TestClient runs the app on its # own. In production both are the single uvicorn loop, which is the arrangement # the in-process registry already requires. async def test_answer_finds_the_pause_and_resolves_it(db, user_id): chat_id, message_id = _chat_that_can_ask(db, user_id) generation = generation_service.Generation(chat_id=chat_id, message_id=message_id) generation.pending = interaction.build("pause-1", [_item()], timeout=30) generation_service._RUNNING[message_id] = generation try: assert generation_service.answer(chat_id, "pause-1", choice="Tea", text="") is True assert (await generation.pending._future).text == "Tea" finally: generation_service._RUNNING.pop(message_id, None) async def test_answer_ignores_a_pause_in_another_chat(db, user_id): """The endpoint checks ownership of the chat, so the lookup is scoped to it -- an id on its own would not be an authorisation.""" chat_id, message_id = _chat_that_can_ask(db, user_id) generation = generation_service.Generation(chat_id=chat_id, message_id=message_id) generation.pending = interaction.build("pause-1", [_item()], timeout=30) generation_service._RUNNING[message_id] = generation try: assert generation_service.answer("another-chat", "pause-1", choice="x", text="") is False assert not generation.pending._future.done() finally: generation_service._RUNNING.pop(message_id, None) async def test_an_allow_choice_is_kept_as_a_verdict_not_as_typed_text(): """"allow" is a decision, not something somebody wrote in the box.""" pause = interaction.build("p", [_item()], timeout=5) pause.resolve(interaction.ALLOW) reply = await pause._future assert reply.permitted is True assert reply.outcome == interaction.ALLOW def test_answering_something_that_has_gone_says_so(client, db, registered, user_id): chat_id, _message_id = _chat_that_can_ask(db, user_id) response = client.post(f"/api/chats/{chat_id}/interaction/nope", data={"choice": "Tea"}) assert response.status_code == 204 assert "no longer waiting" in response.headers["HX-Trigger"] def test_another_account_cannot_answer_your_question(client, db, registered, user_id): """Without the ownership check, a guessed id would be answering -- and later, approving a command in -- somebody else's conversation.""" from lembas.db.models import User chat_id, message_id = _chat_that_can_ask(db, user_id) generation = generation_service.Generation(chat_id=chat_id, message_id=message_id) # No future: the request must be refused before anything tries to resolve # it, so there is nothing here for it to reach. pause = interaction.Interruption(id="pause-2", items=(_item(),)) generation.pending = pause generation_service._RUNNING[message_id] = generation try: client.post("/auth/logout") client.post( "/auth/register", data={"name": "Sam", "email": "sam@shire.test", "password": "correct horse battery"}, follow_redirects=False, ) intruder = db.scalar(select(User).where(User.email == "sam@shire.test")) intruder.role = "user" intruder.active = True db.commit() client.post( "/auth/login", data={"email": "sam@shire.test", "password": "correct horse battery"}, follow_redirects=False, ) response = client.post( f"/api/chats/{chat_id}/interaction/pause-2", data={"choice": "Tea"} ) assert response.status_code == 404 assert pause is generation.pending, "still waiting for the person it belongs to" finally: generation_service._RUNNING.pop(message_id, None) # --- The card ------------------------------------------------------------------ def _render(pending) -> str: from lembas.api.chats import _ask_html return _ask_html("chat-1", pending) def test_nothing_pending_renders_nothing(): """The frame is sent unconditionally so the card can clear itself. An empty string is how it does that.""" assert _render(None) == "" def test_the_card_shows_the_question_and_its_options(): pause = interaction.Interruption( id="p1", items=( interaction.Item( index=0, kind=interaction.KIND_QUESTION, tool_name="ask_user", title="Tea or coffee?", options=("Tea", "Coffee"), ), ), ) html = _render(pause) assert "Tea or coffee?" in html assert 'value="Tea"' in html and 'value="Coffee"' in html assert 'hx-post="/api/chats/chat-1/interaction/p1"' in html assert "The model is asking you" in html, "attributed to the model, not to LLeMbas" def test_the_card_never_offers_a_password_field(): """A model talked into asking for a credential must not be handed a field that looks built for one.""" pause = interaction.Interruption( id="p1", items=( interaction.Item( index=0, kind=interaction.KIND_QUESTION, tool_name="ask_user", title="Confirm your password to continue:", ), ), ) html = _render(pause) assert 'type="password"' not in html assert 'type="text"' in html def test_everything_on_the_card_is_escaped(): """The question is model output, and the model may have been reading somebody else's file a moment ago.""" pause = interaction.Interruption( id="p1", items=( interaction.Item( index=0, kind=interaction.KIND_QUESTION, tool_name="ask_user", title="", detail="rm -rf / ',), ), ), ) html = _render(pause) assert "" not in html assert "<img" in html