HomerandClaude Opus 5
8503c6c775
Three fixes to how a round behaves, found by reading one real round on the live instance rather than by testing it. A member asked "what would you have done differently" answered the person's original question again instead of critiquing what was already there. Fine on a question with one answer; on a request to *make* something it is an invitation. `crowd.turn` now says to respond to what is above and not to re-answer. The model that opened the round, told to write the final answer and take what the others got right, abandoned its own good answer and adopted the newcomer's position with no argument anywhere for why. Both closing fragments now say that an answer is not the worse one for having been written first, and that agreement with no argument behind it is not a reason to change. That second one is not cosmetic: all three answers from the observed round were compiled. The original and the critic's alternative both build; the merged answer that was actually delivered does not. A crowd's failure mode is not looping -- the caps handle that -- it is converging on the last thing said. Third, the reply that opens a round now carries a chip like every other one. It is the single contribution the crowd does not start, so there was nothing to stamp it with until the round began, and a two-model round rendered as an unmarked reply followed by one saying "2 of 2". The stamp is display state and never scheduling state: `crowd.scheduling_state` hides it from everything that decides what happens next, because fed to the scheduler it would inherit the round's clock -- regenerating the opening an hour later would end the round with "out of time" before anybody spoke -- and would hand that reply a member's tools and a member's instruction. And the chip was never translated. It is now, with the count as placeholders rather than three t() calls around one sentence. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
349 lines
12 KiB
Python
349 lines
12 KiB
Python
"""Choosing a crowd, and reading one.
|
|
|
|
Two screens and one rule each. The picker may only ever offer and accept models
|
|
*this person* can reach — a control checked in the template and not in the route is
|
|
advisory, and a crafted request walks past it. The transcript has to say which
|
|
speaker a bubble is and which pass it belongs to, because nine bubbles for one
|
|
question are otherwise indistinguishable from nine people talking at once.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import pytest
|
|
from sqlalchemy import select
|
|
|
|
from lembas.db.models import (
|
|
ROLE_ASSISTANT,
|
|
ROLE_USER,
|
|
Chat,
|
|
Connection,
|
|
Group,
|
|
Model,
|
|
User,
|
|
)
|
|
from lembas.services import chat as chat_service
|
|
from lembas.services import crowd as crowd_service
|
|
from lembas.services import settings_store
|
|
from lembas.services.crypto import encrypt
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def crowd_on(db, registered):
|
|
settings_store.update(db, {"enabled": True}, key=settings_store.CROWD)
|
|
connection = Connection(
|
|
name="Test", base_url="http://127.0.0.1:1", api_key_encrypted=encrypt("")
|
|
)
|
|
db.add(connection)
|
|
db.commit()
|
|
for index, name in enumerate(("main-model", "second-model", "third-model")):
|
|
db.add(
|
|
Model(
|
|
connection_id=connection.id,
|
|
model_id=name,
|
|
display_name=name,
|
|
position=index,
|
|
capabilities_json={"tools": True},
|
|
)
|
|
)
|
|
db.commit()
|
|
|
|
|
|
def _user(db) -> User:
|
|
return db.scalars(select(User).order_by(User.created_at)).first()
|
|
|
|
|
|
def _chat(db) -> Chat:
|
|
connection = db.scalars(select(Connection)).first()
|
|
chat = Chat(
|
|
user_id=_user(db).id, title="t", model_id="main-model", connection_id=connection.id
|
|
)
|
|
db.add(chat)
|
|
db.commit()
|
|
return chat
|
|
|
|
|
|
def _members(db, chat) -> list[str]:
|
|
db.expire_all()
|
|
return [
|
|
row.model_id
|
|
for row in sorted(db.get(Chat, chat.id).crowd, key=lambda r: r.position)
|
|
]
|
|
|
|
|
|
# --- Reachable where somebody would look --------------------------------------
|
|
#
|
|
# The feature shipped in 1.6.0 switched on and unreachable: the only control was
|
|
# inside the Chat settings panel, behind the ⋯ menu, in a chat that already
|
|
# existed. The owner enabled it, went looking, and reported that there was nothing
|
|
# to find. A control nobody can find is a feature nobody has, so these assert the
|
|
# two places it has to be rather than the one place it was.
|
|
def test_the_composer_offers_the_crowd_in_a_chat(client, db):
|
|
"""Beside the tool switches, where the comparable decisions are."""
|
|
chat = _chat(db)
|
|
page = client.get(f"/chat/{chat.id}").text
|
|
assert 'name="crowd_model_ids"' in page
|
|
assert 'form="crowd-form"' in page
|
|
assert '<form id="crowd-form">' in page
|
|
|
|
|
|
def test_the_composer_offers_the_crowd_before_the_chat_exists(client, db):
|
|
"""On the new-chat screen there is no row to attach anybody to, so the choice
|
|
rides along with the first message — the mechanism the scope switches use."""
|
|
page = client.get("/chat").text
|
|
assert 'name="crowd_model_ids"' in page
|
|
# Riding along, so no sibling form and no PATCH: the composer's own POST
|
|
# carries it.
|
|
assert '<form id="crowd-form">' not in page
|
|
assert 'value="second-model"' in page
|
|
|
|
|
|
def test_starting_a_chat_with_a_crowd_keeps_it(client, db):
|
|
"""The end of that path: the first message creates the chat *and* its crowd."""
|
|
from sqlalchemy import select as sa_select
|
|
|
|
client.post(
|
|
"/api/chats/start",
|
|
data={
|
|
"content": "Who is right?",
|
|
"model_id": "main-model",
|
|
"crowd_model_ids": ["", "second-model", "third-model"],
|
|
},
|
|
follow_redirects=False,
|
|
)
|
|
db.expire_all()
|
|
chat = db.scalars(sa_select(Chat).order_by(Chat.created_at.desc())).first()
|
|
assert [row.model_id for row in sorted(chat.crowd, key=lambda r: r.position)] == [
|
|
"second-model",
|
|
"third-model",
|
|
]
|
|
|
|
|
|
def test_starting_a_chat_refuses_a_model_the_person_cannot_reach(client, db):
|
|
"""The same rule as the panel, in the same one function, so there is nowhere
|
|
for the two to disagree."""
|
|
from sqlalchemy import select as sa_select
|
|
|
|
group = Group(name="Wheel")
|
|
db.add(group)
|
|
restricted = db.scalar(select(Model).where(Model.model_id == "third-model"))
|
|
restricted.public = False
|
|
restricted.groups = [group]
|
|
user = _user(db)
|
|
user.role = "user"
|
|
db.commit()
|
|
|
|
client.post(
|
|
"/api/chats/start",
|
|
data={
|
|
"content": "Who is right?",
|
|
"model_id": "main-model",
|
|
"crowd_model_ids": ["second-model", "third-model"],
|
|
},
|
|
follow_redirects=False,
|
|
)
|
|
db.expire_all()
|
|
chat = db.scalars(sa_select(Chat).order_by(Chat.created_at.desc())).first()
|
|
assert [row.model_id for row in chat.crowd] == ["second-model"]
|
|
|
|
|
|
def test_the_composer_control_is_absent_while_the_feature_is_off(client, db):
|
|
settings_store.update(db, {"enabled": False}, key=settings_store.CROWD)
|
|
assert 'name="crowd_model_ids"' not in client.get("/chat").text
|
|
|
|
|
|
# --- Choosing -----------------------------------------------------------------
|
|
def test_the_panel_offers_the_other_models(client, db):
|
|
chat = _chat(db)
|
|
page = client.get(f"/chat/{chat.id}").text
|
|
assert 'name="crowd_model_ids"' in page
|
|
assert 'name="crowd_model_ids" value="second-model"' in page
|
|
# Never the chat's own model: it would answer twice in a row. Asserted with
|
|
# the field name attached, because the model *picker* on the same page quite
|
|
# correctly offers it.
|
|
assert 'name="crowd_model_ids" value="main-model"' not in page
|
|
|
|
|
|
def test_the_panel_is_absent_while_the_feature_is_off(client, db):
|
|
settings_store.update(db, {"enabled": False}, key=settings_store.CROWD)
|
|
chat = _chat(db)
|
|
assert 'name="crowd_model_ids"' not in client.get(f"/chat/{chat.id}").text
|
|
|
|
|
|
def test_ticking_a_model_adds_it_in_order(client, db):
|
|
chat = _chat(db)
|
|
client.patch(
|
|
f"/api/chats/{chat.id}",
|
|
data={"crowd_model_ids": ["second-model", "third-model"]},
|
|
)
|
|
assert _members(db, chat) == ["second-model", "third-model"]
|
|
|
|
|
|
def test_clearing_every_box_clears_the_crowd(client, db):
|
|
"""The single field always sent is what makes this possible: an absent
|
|
checkbox carries no signal of its own."""
|
|
chat = _chat(db)
|
|
client.patch(f"/api/chats/{chat.id}", data={"crowd_model_ids": ["second-model"]})
|
|
assert _members(db, chat) == ["second-model"]
|
|
|
|
client.patch(f"/api/chats/{chat.id}", data={"crowd_model_ids": [""]})
|
|
assert _members(db, chat) == []
|
|
|
|
|
|
def test_a_model_this_person_cannot_reach_is_refused(client, db):
|
|
"""Checked in the route, not only in the template. Otherwise the picker is
|
|
advisory."""
|
|
group = Group(name="Wheel")
|
|
db.add(group)
|
|
restricted = db.scalar(select(Model).where(Model.model_id == "third-model"))
|
|
restricted.public = False
|
|
restricted.groups = [group]
|
|
user = _user(db)
|
|
user.role = "user"
|
|
db.commit()
|
|
chat = _chat(db)
|
|
|
|
client.patch(
|
|
f"/api/chats/{chat.id}",
|
|
data={"crowd_model_ids": ["second-model", "third-model"]},
|
|
)
|
|
|
|
assert _members(db, chat) == ["second-model"]
|
|
|
|
|
|
def test_the_chats_own_model_cannot_be_added(client, db):
|
|
chat = _chat(db)
|
|
client.patch(f"/api/chats/{chat.id}", data={"crowd_model_ids": ["main-model"]})
|
|
assert _members(db, chat) == []
|
|
|
|
|
|
def test_the_same_model_twice_is_one_member(client, db):
|
|
chat = _chat(db)
|
|
client.patch(
|
|
f"/api/chats/{chat.id}", data={"crowd_model_ids": ["second-model", "second-model"]}
|
|
)
|
|
assert _members(db, chat) == ["second-model"]
|
|
|
|
|
|
def test_the_cap_trims_what_is_accepted(client, db):
|
|
settings_store.update(db, {"max_models": 1}, key=settings_store.CROWD)
|
|
chat = _chat(db)
|
|
client.patch(
|
|
f"/api/chats/{chat.id}", data={"crowd_model_ids": ["second-model", "third-model"]}
|
|
)
|
|
assert _members(db, chat) == ["second-model"]
|
|
|
|
|
|
def test_the_panel_says_what_a_turn_will_cost(client, db):
|
|
"""The thing somebody will not have thought about: a turn is
|
|
speakers x rounds x 2 - 1 replies, and each is a whole reply."""
|
|
chat = _chat(db)
|
|
client.patch(
|
|
f"/api/chats/{chat.id}", data={"crowd_model_ids": ["second-model", "third-model"]}
|
|
)
|
|
page = client.get(f"/chat/{chat.id}").text
|
|
assert "5 replies a turn" in page
|
|
|
|
|
|
def test_a_member_that_can_no_longer_be_reached_is_shown_struck_through(client, db):
|
|
"""Membership is text with no foreign key, so the row outlives the model. Saying
|
|
so beats both deleting it and pretending it still answers."""
|
|
chat = _chat(db)
|
|
client.patch(f"/api/chats/{chat.id}", data={"crowd_model_ids": ["third-model"]})
|
|
gone = db.scalar(select(Model).where(Model.model_id == "third-model"))
|
|
db.delete(gone)
|
|
db.commit()
|
|
|
|
page = client.get(f"/chat/{chat.id}").text
|
|
|
|
assert "Skipped" in page
|
|
assert "<s>third-model</s>" in page
|
|
|
|
|
|
# --- Reading ------------------------------------------------------------------
|
|
def _bubble(db, chat, *, phase, index=1, of=3, stopped="", round_=1) -> str:
|
|
from lembas.api import chats as chats_api
|
|
|
|
chat_service.create_message(db, chat, ROLE_USER, "What should we do?")
|
|
message = chat_service.create_message(
|
|
db, chat, ROLE_ASSISTANT, "Something.", model_id="second-model"
|
|
)
|
|
message.crowd_json = crowd_service.Turn(
|
|
turn="u1", round=round_, phase=phase, index=index, of=of,
|
|
started_at=crowd_service.now_stamp(), stopped=stopped,
|
|
).as_json()
|
|
db.commit()
|
|
return chats_api._render_bubble(db, chat, _user(db), message)
|
|
|
|
|
|
def test_a_bubble_on_the_way_out_says_which_speaker_it_is(db):
|
|
chat = _chat(db)
|
|
html = _bubble(db, chat, phase=crowd_service.PHASE_OUT, index=1, of=3)
|
|
assert "2 of 3" in html
|
|
|
|
|
|
def test_the_bubble_that_opened_the_round_says_it_is_first(db):
|
|
"""The opening reply is stamped once the round begins, so it says `1 of 3`.
|
|
|
|
Before that it was the one contribution with no chip at all, which made a
|
|
two-model round read as an ordinary answer followed by one labelled `2 of 2`.
|
|
"""
|
|
chat = _chat(db)
|
|
html = _bubble(db, chat, phase=crowd_service.PHASE_OUT, index=0, of=3)
|
|
assert "1 of 3" in html
|
|
|
|
|
|
def test_the_chip_is_translated(db):
|
|
"""It is prose a person reads, and it was English on a Slovak instance."""
|
|
from lembas.web import i18n
|
|
|
|
chat = _chat(db)
|
|
i18n.activate("sk")
|
|
try:
|
|
html = _bubble(db, chat, phase=crowd_service.PHASE_BACK, index=1, of=3)
|
|
finally:
|
|
i18n.activate("en")
|
|
assert "na ceste späť" in html
|
|
assert "on the way back" not in html
|
|
|
|
|
|
def test_a_bubble_on_the_way_back_says_so_and_is_quieter(db):
|
|
chat = _chat(db)
|
|
html = _bubble(db, chat, phase=crowd_service.PHASE_BACK)
|
|
assert "on the way back" in html
|
|
assert "msg--crowd-back" in html
|
|
|
|
|
|
def test_the_closing_bubble_says_it_is_closing(db):
|
|
chat = _chat(db)
|
|
html = _bubble(db, chat, phase=crowd_service.PHASE_CLOSE, index=0)
|
|
assert "closing" in html
|
|
|
|
|
|
def test_a_later_round_is_numbered(db):
|
|
chat = _chat(db)
|
|
html = _bubble(db, chat, phase=crowd_service.PHASE_OUT, round_=2)
|
|
assert "round 2" in html
|
|
|
|
|
|
def test_why_a_round_ended_is_shown_where_it_ended(db):
|
|
"""Otherwise a crowd that ran out of rounds or time simply stops, which reads
|
|
as the feature failing rather than as a limit doing its job."""
|
|
chat = _chat(db)
|
|
assert "no rounds left" in _bubble(
|
|
db, chat, phase=crowd_service.PHASE_CLOSE, stopped=crowd_service.STOPPED_ROUNDS
|
|
)
|
|
|
|
|
|
def test_an_ordinary_bubble_carries_no_crowd_chip(db):
|
|
from lembas.api import chats as chats_api
|
|
|
|
chat = _chat(db)
|
|
chat_service.create_message(db, chat, ROLE_USER, "Hello")
|
|
message = chat_service.create_message(
|
|
db, chat, ROLE_ASSISTANT, "Hello back.", model_id="main-model"
|
|
)
|
|
html = chats_api._render_bubble(db, chat, _user(db), message)
|
|
assert "of 3" not in html
|
|
assert "msg--crowd-back" not in html
|
|
assert "closing" not in html
|