"""What one crowd speaker is actually sent. Pure: `build_request` with a speaker, no generation and no endpoint. Two properties matter more than anything else here, and both are silent when wrong. **Another speaker's reply must not arrive as this one's own turn.** Sent verbatim, every assistant message in the payload reads as something *this* model wrote — so it defends sentences it never said and cannot disagree with them, which is the entire purpose of the backward pass. **The history has to alternate.** Several chat templates reject one that does not, and this project already works around it once: `task.compact_ack` exists so a compacted history still alternates. A crowd produces consecutive assistant turns by construction, so relabelling is what keeps it sendable at all. """ from __future__ import annotations import pytest from sqlalchemy import select from lembas.db.models import ROLE_ASSISTANT, ROLE_USER, Chat, Connection, Model, User from lembas.services import chat as chat_service from lembas.services import crowd as crowd_service from lembas.services import prompts as prompts_service from lembas.services.crypto import encrypt MODELS = ("main-model", "second-model", "third-model") @pytest.fixture(autouse=True) def three_models(db, registered): connection = Connection( name="Test", base_url="http://127.0.0.1:1", api_key_encrypted=encrypt("") ) db.add(connection) db.commit() for index, name in enumerate(MODELS): db.add( Model( connection_id=connection.id, model_id=name, display_name=name.replace("-model", "").title(), position=index, capabilities_json={"tools": True}, ) ) db.commit() def _user(db) -> User: return db.scalars(select(User).order_by(User.created_at)).first() def _chat(db) -> Chat: connection = db.scalars(select(Connection)).first() chat = Chat( user_id=_user(db).id, title="t", model_id="main-model", connection_id=connection.id ) db.add(chat) db.commit() return chat def _round(db, chat, answers: list[tuple[str, str]]): """A user turn, then one assistant reply per (model, text).""" chat_service.create_message(db, chat, ROLE_USER, "What should we do?") for model_id, text in answers: chat_service.create_message(db, chat, ROLE_ASSISTANT, text, model_id=model_id) def _payload(db, chat, speaker_id: str, *, turn=None, again=False): placeholder = chat_service.create_message( db, chat, ROLE_ASSISTANT, "", complete_=False, model_id=speaker_id ) if turn is not None: placeholder.crowd_json = turn.as_json() db.commit() body = chat_service.build_request( db, chat, upto=placeholder, user=_user(db), crowd_again=again ) return body["messages"] def _roles(messages) -> list[str]: return [m["role"] for m in messages if m["role"] != "system"] # --- Whose words are whose ---------------------------------------------------- def test_another_speakers_answer_arrives_quoted_and_attributed(db): chat = _chat(db) _round(db, chat, [("main-model", "Rewrite it in Rust.")]) messages = _payload(db, chat, "second-model") quoted = [m for m in messages if "Rewrite it in Rust." in str(m["content"])] assert quoted, "the other speaker's answer never reached this one" assert quoted[0]["role"] == ROLE_USER, "it arrived as this model's own words" assert "Main" in quoted[0]["content"], "it arrived unattributed" def test_a_speakers_own_earlier_turn_stays_its_own(db): """Relabelling everything would be the same bug from the other side: a model told that its own answer was somebody else's cannot be held to it.""" chat = _chat(db) _round(db, chat, [("main-model", "Mine."), ("second-model", "Theirs.")]) messages = _payload(db, chat, "main-model") mine = [m for m in messages if "Mine." in str(m["content"])] assert mine[0]["role"] == ROLE_ASSISTANT theirs = [m for m in messages if "Theirs." in str(m["content"])] assert theirs[0]["role"] == ROLE_USER def test_an_ordinary_one_model_chat_is_untouched(db): """A chat with no other speaker in it must build the payload it always did. Asserted as the property rather than by comparing two calls: `build_request` resolves the harness and a bare `build_messages` does not, so comparing the two would fail for a reason that has nothing to do with crowds -- which is what the first version of this test did. """ chat = _chat(db) _round(db, chat, [("main-model", "Just me.")]) messages = _payload(db, chat, "main-model") assert _roles(messages) == [ROLE_USER, ROLE_ASSISTANT] assert all("answered:" not in str(m["content"]) for m in messages) # And nothing was appended: no crowd state on the row means no instruction. assert messages[-1]["content"] == "Just me." # --- Alternation -------------------------------------------------------------- @pytest.mark.parametrize("speaker_id", MODELS) def test_no_two_turns_in_a_row_share_a_role(db, speaker_id): """The property, for every speaker in a three-model round. A run of assistant turns is what a crowd produces naturally and what templates refuse.""" chat = _chat(db) _round( db, chat, [("main-model", "One."), ("second-model", "Two."), ("third-model", "Three.")], ) roles = _roles(_payload(db, chat, speaker_id)) assert all(a != b for a, b in zip(roles, roles[1:], strict=False)), roles def test_the_history_still_starts_on_a_user_turn(db): """What every chat template expects, and what the compaction pair exists to preserve.""" chat = _chat(db) _round(db, chat, [("main-model", "One."), ("second-model", "Two.")]) assert _roles(_payload(db, chat, "third-model"))[0] == ROLE_USER def test_a_relabelled_turn_never_becomes_multimodal(db): """Built directly rather than by calling `message_payload` with a swapped role: that one attaches image parts when the role is `user`, so a swapped assistant turn carrying a generated image would silently become a content list -- and an endpoint that rejects one rejects every later turn with it.""" chat = _chat(db) _round(db, chat, [("main-model", "Here is a picture.")]) messages = _payload(db, chat, "second-model") for entry in messages: assert isinstance(entry["content"], str), entry # --- The instruction ---------------------------------------------------------- def _turn(phase, index=1, of=3): return crowd_service.Turn( turn="u1", round=1, phase=phase, index=index, of=of, started_at=crowd_service.now_stamp(), ) def test_the_forward_pass_asks_for_what_is_missing(db): chat = _chat(db) _round(db, chat, [("main-model", "One.")]) messages = _payload(db, chat, "second-model", turn=_turn(crowd_service.PHASE_OUT)) assert "Add what is missing" in messages[-1]["content"] assert messages[-1]["role"] == ROLE_USER def test_the_way_back_asks_for_disagreement(db): chat = _chat(db) _round(db, chat, [("main-model", "One."), ("second-model", "Two.")]) messages = _payload(db, chat, "second-model", turn=_turn(crowd_service.PHASE_BACK)) assert "disagree" in messages[-1]["content"] def test_the_closing_turn_offers_another_round_only_when_there_is_one(db): chat = _chat(db) _round(db, chat, [("main-model", "One."), ("second-model", "Two.")]) with_tool = _payload( db, chat, "main-model", turn=_turn(crowd_service.PHASE_CLOSE, index=0), again=True ) assert "crowd_again" in with_tool[-1]["content"] without = _payload( db, chat, "main-model", turn=_turn(crowd_service.PHASE_CLOSE, index=0), again=False ) assert "crowd_again" not in without[-1]["content"] assert "no further round" in without[-1]["content"] def test_the_instruction_is_not_written_into_the_transcript(db): """Payload only. A row would double the bubbles, would be answered by every later speaker as an ordinary user turn, and could be dropped from the request entirely by a `created_at` tie with the placeholder.""" from lembas.db.models import Message chat = _chat(db) _round(db, chat, [("main-model", "One.")]) before = db.scalar(select(Message).order_by(Message.created_at.desc())) _payload(db, chat, "second-model", turn=_turn(crowd_service.PHASE_OUT)) db.expire_all() rows = db.scalars(select(Message).where(Message.chat_id == chat.id)).all() assert not any( "Add what is missing" in (row.content or "") for row in rows ), "the instruction was written into the conversation" assert before is not None def test_clearing_the_fragment_sends_no_instruction(db): """An administrator emptying a fragment is switching that wording off, which is the convention everywhere else here -- and an empty user turn is not a thing to send.""" prompts_service.save(db, {"crowd.turn": ""}) chat = _chat(db) _round(db, chat, [("main-model", "One.")]) messages = _payload(db, chat, "second-model", turn=_turn(crowd_service.PHASE_OUT)) assert messages[-1]["role"] == ROLE_USER assert "Add what is missing" not in messages[-1]["content"] def test_the_instruction_merges_rather_than_doubling_a_user_turn(db): """It lands after a quoted answer, which is itself a user turn now.""" chat = _chat(db) _round(db, chat, [("main-model", "One.")]) roles = _roles(_payload(db, chat, "second-model", turn=_turn(crowd_service.PHASE_OUT))) assert all(a != b for a, b in zip(roles, roles[1:], strict=False)), roles