Houmeres/LLeMbas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

Ask several questions on one card

Browse Source 
One `ask_user` call can now carry several questions, and they come back in a
single submit. Asking one at a time cost a round trip and an interruption
each, and by the third you had forgotten the first.

Each question becomes an item with its own key; several items share a call
index, because they belong to one call and one tool turn has to answer them
all. Each answer is quoted beside the question it belongs to -- with four on
a card, a bare list would leave the model matching them up by position and
sometimes getting it wrong.

Options are radios rather than submit buttons, so picking one does not send
the form while two other questions are still blank. What you type beats what
you picked: someone who writes in the box after clicking an option meant the
writing.

`_questions_in` also reads the shapes a small model actually sends -- a bare
`question` string, a list of plain strings, one object where a list belonged.
Getting that wrong costs a whole round trip and shows a card saying nothing.

Two test fixes, both mine. `test_posting_a_message_stores_both_turns` raced
the background generation it started: against a connection that refuses
instantly the reply sometimes won, writing the error and marking the row
complete before the assertions could read it. And the generation registry is
module-global, so a test that started a reply left an entry -- and a Task
belonging to a closed event loop -- for the rest of the session.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
2026-08-01 22:01:46 +02:00
parent 8c3fe97939
commit 671e49cae8
10 changed files with 673 additions and 138 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_agent_interaction.py
+311 -31
View File
@@ -137,10 +137,10 @@ async def test_the_reply_pauses_and_the_card_describes_the_question(scripted):
pending = await _until_paused(generation)
assert pending.kind == interaction.KIND_QUESTION
assert pending.items[0].title == "Tea or coffee?"
assert pending.options == ("Tea", "Coffee")
assert pending.items[0].options == ("Tea", "Coffee")
assert "Waiting for your answer" in generation.status
pending.resolve(interaction.ANSWER, text="Tea")
pending.resolve(interaction.ANSWER, answers={"q0": "Tea"})
await task
@@ -149,7 +149,7 @@ async def test_the_answer_reaches_the_model_as_a_tool_result(scripted):
task = asyncio.create_task(generation_service._run(generation))
pending = await _until_paused(generation)
pending.resolve(interaction.ANSWER, text="Coffee, please")
pending.resolve(interaction.ANSWER, answers={"q0": "Coffee, please"})
await task
turns = [m for m in payloads[1]["messages"] if m.get("role") == "tool"]
@@ -164,7 +164,7 @@ async def test_the_card_is_cleared_once_it_is_answered(scripted):
pending = await _until_paused(generation)
version = generation.version
pending.resolve(interaction.ANSWER, text="Tea")
pending.resolve(interaction.ANSWER, answers={"q0": "Tea"})
await task
assert generation.pending is None
@@ -176,13 +176,13 @@ async def test_the_transcript_keeps_what_was_asked_and_answered(scripted):
task = asyncio.create_task(generation_service._run(generation))
pending = await _until_paused(generation)
pending.resolve(interaction.ANSWER, text="Tea")
pending.resolve(interaction.ANSWER, answers={"q0": "Tea"})
await task
event = generation.tool_events[0]
assert event["kind"] == "ask"
assert event["query"] == "Tea or coffee?"
assert event["text"] == "Tea"
assert "Tea" in event["text"]
# --- Stop, while nothing is streaming ----------------------------------------
@@ -249,9 +249,9 @@ def test_resolving_twice_only_counts_once():
async def go():
pause = interaction.build("abc", [_item()], timeout=5)
assert pause.resolve(interaction.ANSWER, text="first") is True
assert pause.resolve(interaction.ANSWER, text="second") is False
assert (await pause._future).text == "first"
assert pause.resolve(interaction.ANSWER, answers={"q0": "first"}) is True
assert pause.resolve(interaction.ANSWER, answers={"q0": "second"}) is False
assert (await pause._future).answers == {"q0": "first"}
asyncio.run(go())
@@ -261,12 +261,14 @@ def test_an_interruption_with_no_future_cannot_be_resolved():
assert pause.resolve(interaction.ANSWER) is False
def _item() -> interaction.Item:
def _item(key: str = "q0", *, title: str = "Tea or coffee?", **kwargs) -> interaction.Item:
return interaction.Item(
index=0,
key=key,
kind=interaction.KIND_QUESTION,
tool_name="ask_user",
title="Tea or coffee?",
title=title,
**kwargs,
)
@@ -281,8 +283,8 @@ async def test_answer_finds_the_pause_and_resolves_it(db, user_id):
generation.pending = interaction.build("pause-1", [_item()], timeout=30)
generation_service._RUNNING[message_id] = generation
try:
assert generation_service.answer(chat_id, "pause-1", choice="Tea", text="") is True
assert (await generation.pending._future).text == "Tea"
assert generation_service.answer(chat_id, "pause-1", answers={"q0": "Tea"}) is True
assert (await generation.pending._future).answers == {"q0": "Tea"}
finally:
generation_service._RUNNING.pop(message_id, None)
@@ -295,7 +297,7 @@ async def test_answer_ignores_a_pause_in_another_chat(db, user_id):
generation.pending = interaction.build("pause-1", [_item()], timeout=30)
generation_service._RUNNING[message_id] = generation
try:
assert generation_service.answer("another-chat", "pause-1", choice="x", text="") is False
assert generation_service.answer("another-chat", "pause-1", answers={"q0": "x"}) is False
assert not generation.pending._future.done()
finally:
generation_service._RUNNING.pop(message_id, None)
@@ -372,13 +374,7 @@ def test_the_card_shows_the_question_and_its_options():
pause = interaction.Interruption(
id="p1",
items=(
interaction.Item(
index=0,
kind=interaction.KIND_QUESTION,
tool_name="ask_user",
title="Tea or coffee?",
options=("Tea", "Coffee"),
),
_item(title="Tea or coffee?", options=("Tea", "Coffee")),
),
)
html = _render(pause)
@@ -394,12 +390,7 @@ def test_the_card_never_offers_a_password_field():
pause = interaction.Interruption(
id="p1",
items=(
interaction.Item(
index=0,
kind=interaction.KIND_QUESTION,
tool_name="ask_user",
title="Confirm your password to continue:",
),
_item(title="Confirm your password to continue:"),
),
)
html = _render(pause)
@@ -413,10 +404,7 @@ def test_everything_on_the_card_is_escaped():
pause = interaction.Interruption(
id="p1",
items=(
interaction.Item(
index=0,
kind=interaction.KIND_QUESTION,
tool_name="ask_user",
_item(
title="<img src=x onerror=alert(1)>",
detail="rm -rf / <script>",
options=('"><script>bad</script>',),
@@ -427,3 +415,295 @@ def test_everything_on_the_card_is_escaped():
assert "<img" not in html
assert "<script>" not in html
assert "&lt;img" in html
# --- Several questions, one card, one submit ---------------------------------
def _multi_chunk(questions: list[dict], *, call_id="c1", index=0):
import json as _json
return {
"choices": [
{
"delta": {
"tool_calls": [
{
"index": index,
"id": call_id,
"function": {
"name": "ask_user",
"arguments": _json.dumps({"questions": questions}),
},
}
]
}
}
]
}
async def test_several_questions_arrive_on_one_card(db, user_id, monkeypatch):
chat_id, message_id = _chat_that_can_ask(db, user_id)
monkeypatch.setattr(
generation_service,
"stream_chat",
_stub_stream(
[
[
_multi_chunk(
[
{"question": "Which database?", "options": ["SQLite", "Postgres"]},
{"question": "Which port?"},
{"question": "Deploy now?", "options": ["Yes", "Later"]},
]
)
],
[_text_chunk("Understood.")],
],
[],
),
)
generation = generation_service.Generation(chat_id=chat_id, message_id=message_id)
task = asyncio.create_task(generation_service._run(generation))
pending = await _until_paused(generation)
assert len(pending.items) == 3
assert [i.title for i in pending.items] == ["Which database?", "Which port?", "Deploy now?"]
assert [i.key for i in pending.items] == ["q0", "q1", "q2"]
assert pending.items[1].options == (), "a question may have no options at all"
assert "Waiting for your answers" in generation.status
pending.resolve(interaction.ANSWER, answers={"q0": "Postgres", "q1": "5433", "q2": "Later"})
await task
async def test_all_the_answers_come_back_in_one_tool_turn(db, user_id, monkeypatch):
"""One call, one turn -- however many questions it carried. The endpoint
expects exactly one tool result per tool_call_id."""
chat_id, message_id = _chat_that_can_ask(db, user_id)
payloads: list[dict] = []
monkeypatch.setattr(
generation_service,
"stream_chat",
_stub_stream(
[
[_multi_chunk([{"question": "Which database?"}, {"question": "Which port?"}])],
[_text_chunk("Understood.")],
],
payloads,
),
)
generation = generation_service.Generation(chat_id=chat_id, message_id=message_id)
task = asyncio.create_task(generation_service._run(generation))
pending = await _until_paused(generation)
pending.resolve(interaction.ANSWER, answers={"q0": "Postgres", "q1": "5433"})
await task
turns = [m for m in payloads[1]["messages"] if m.get("role") == "tool"]
assert len(turns) == 1
content = turns[0]["content"]
# Each answer is quoted beside its own question, so the model is not left
# matching them up by position.
assert "Which database?" in content and "Postgres" in content
assert "Which port?" in content and "5433" in content
assert content.index("Which database?") < content.index("Which port?")
async def test_a_question_left_blank_is_reported_as_skipped(db, user_id, monkeypatch):
chat_id, message_id = _chat_that_can_ask(db, user_id)
payloads: list[dict] = []
monkeypatch.setattr(
generation_service,
"stream_chat",
_stub_stream(
[
[_multi_chunk([{"question": "Which database?"}, {"question": "Which port?"}])],
[_text_chunk("Understood.")],
],
payloads,
),
)
generation = generation_service.Generation(chat_id=chat_id, message_id=message_id)
task = asyncio.create_task(generation_service._run(generation))
pending = await _until_paused(generation)
pending.resolve(interaction.ANSWER, answers={"q0": "Postgres", "q1": ""})
await task
content = [m for m in payloads[1]["messages"] if m.get("role") == "tool"][0]["content"]
assert "Postgres" in content
assert "left unanswered" in content and "Which port?" in content
async def test_two_ask_calls_in_one_round_share_a_card_but_answer_separately(
db, user_id, monkeypatch
):
"""One card, because the reader should be interrupted once -- but two tool
turns, because there were two calls."""
import json as _json
chat_id, message_id = _chat_that_can_ask(db, user_id)
payloads: list[dict] = []
both = {
"choices": [
{
"delta": {
"tool_calls": [
{
"index": 0,
"id": "a",
"function": {
"name": "ask_user",
"arguments": _json.dumps({"questions": [{"question": "First?"}]}),
},
},
{
"index": 1,
"id": "b",
"function": {
"name": "ask_user",
"arguments": _json.dumps({"questions": [{"question": "Second?"}]}),
},
},
]
}
}
]
}
monkeypatch.setattr(
generation_service,
"stream_chat",
_stub_stream([[both], [_text_chunk("Understood.")]], payloads),
)
generation = generation_service.Generation(chat_id=chat_id, message_id=message_id)
task = asyncio.create_task(generation_service._run(generation))
pending = await _until_paused(generation)
assert len(pending.items) == 2
assert {i.index for i in pending.items} == {0, 1}, "one item per call"
assert [i.key for i in pending.items] == ["q0", "q1"], "keys are unique across calls"
pending.resolve(interaction.ANSWER, answers={"q0": "one", "q1": "two"})
await task
turns = [m for m in payloads[1]["messages"] if m.get("role") == "tool"]
assert [t["tool_call_id"] for t in turns] == ["a", "b"]
assert "one" in turns[0]["content"] and "two" in turns[1]["content"]
# --- Whatever shape the model actually emits ----------------------------------
@pytest.mark.parametrize(
("args", "expected"),
[
({"questions": [{"question": "A"}, {"question": "B"}]}, ["A", "B"]),
({"question": "A"}, ["A"]), # the singular form
({"questions": "A"}, ["A"]), # a string where a list belonged
({"questions": ["A", "B"]}, ["A", "B"]), # bare strings
({"questions": {"question": "A"}}, ["A"]), # one object, not wrapped
({"questions": [{"nope": 1}, {"question": "B"}]}, ["B"]), # junk is dropped
({}, []),
({"questions": []}, []),
],
)
def test_the_questions_are_read_however_they_were_spelled(args, expected):
"""A capable model sends the schema. A small one sends something close, and
getting it wrong costs a round trip and shows a card saying nothing."""
asked = generation_service._questions_in(args)
assert [q["question"] for q in asked] == expected
def test_a_lone_bare_question_keeps_a_sibling_options_list():
asked = generation_service._questions_in({"questions": ["Tea or coffee?"], "options": ["Tea"]})
assert asked[0]["options"] == ["Tea"]
def test_too_many_questions_are_cut_off():
args = {"questions": [{"question": f"Q{i}"} for i in range(20)]}
assert len(generation_service._questions_in(args)) == interaction.MAX_QUESTIONS
def test_the_card_renders_every_question_with_its_own_fields():
pause = interaction.Interruption(
id="p1",
items=(
_item("q0", title="Which database?", options=("SQLite", "Postgres")),
_item("q1", title="Which port?"),
),
)
html = _render(pause)
assert "Which database?" in html and "Which port?" in html
# Radios rather than submit buttons: picking one must not send the form
# while two other questions are still blank.
assert 'type="radio" name="choice.q0" value="SQLite"' in html
assert 'name="text.q0"' in html and 'name="text.q1"' in html
# A question with no options still gets somewhere to write.
assert 'name="choice.q1"' not in html
assert html.count("Send answers") == 1, "one submit for the whole card"
def test_a_single_question_says_answer_rather_than_send_answers():
html = _render(interaction.Interruption(id="p1", items=(_item(),)))
assert "Send answers" not in html
assert ">\n Answer" in html or "Answer" in html
def test_the_endpoint_gathers_every_answer_at_once(client, db, registered, user_id):
"""The whole card in one POST -- what was typed beating what was picked."""
chat_id, message_id = _chat_that_can_ask(db, user_id)
seen: dict = {}
def capture(chat, interaction_id, *, verdict="", answers=None):
seen["chat"] = chat
seen["id"] = interaction_id
seen["verdict"] = verdict
seen["answers"] = answers
return True
from lembas.api import chats as chats_api
original = chats_api.generation_service.answer
chats_api.generation_service.answer = capture
try:
response = client.post(
f"/api/chats/{chat_id}/interaction/p1",
data={
"choice.q0": "Postgres",
"text.q0": "",
"choice.q1": "Yes",
"text.q1": "actually, later",
"text.q2": "5433",
},
)
assert response.status_code == 204
finally:
chats_api.generation_service.answer = original
assert seen["answers"] == {
"q0": "Postgres", # picked, nothing written
"q1": "actually, later", # written wins over picked
"q2": "5433", # written, nothing to pick
}
assert seen["verdict"] == ""
assert message_id # the chat was real
def test_the_endpoint_passes_a_verdict_through_untouched(client, db, registered, user_id):
chat_id, _message_id = _chat_that_can_ask(db, user_id)
seen: dict = {}
from lembas.api import chats as chats_api
original = chats_api.generation_service.answer
chats_api.generation_service.answer = lambda c, i, *, verdict="", answers=None: (
seen.update(verdict=verdict, answers=answers) or True
)
try:
client.post(f"/api/chats/{chat_id}/interaction/p1", data={"verdict": "allow"})
finally:
chats_api.generation_service.answer = original
assert seen["verdict"] == "allow"
assert seen["answers"] == {}