Refusing can say why, and the why is an instruction
"Don't" told the model it was refused and nothing else, so it did the one sensible thing left and asked what you would rather -- a whole round spent on something you knew when you pressed the button. "Give reason" opens a box beside it, and what you write goes back with the refusal. The reason changes what the model is *told*, not only what it reads, and that is the whole of the feature. `_not_allowed` branches: given nothing to go on, "say what you were going to do and ask what they would prefer" is right; given a reason it is exactly wrong, because the answer is already on the screen above and the model spends a round asking for it again. So it is pointed at the reason and told to carry on from it. The "do not look for a way round" half is kept either way -- that half is about the refusal and holds regardless. A card-level field rather than `text.<key>`. One card covers everything in the round for the reason the primitive exists, so one reason answers the round; and on an approval card `text.<key>` already means a corrected command, which is a different thing arriving in the same shape. Read only on a refusal, so a reason typed and then abandoned by pressing Allow cannot travel with a permission. Bounded where the Reply is built, so nothing downstream thinks about length, and put on the tool event as well as in the result -- a transcript saying a step was refused without saying why is one you had to have been watching to understand. It is also the one thing in a tool result that is genuinely not untrusted: the reader's own words, stated as theirs, needing no fence. Both halves of the control are in the DOM with one hidden and the textarea disabled while hidden, which is the rule the edit box beside it already states: a field created by a click submits nothing when the click handler fails, and an empty `reason` arriving would have to be told from one somebody cleared. The version bump is not incidental. chat.css changed and the service worker caches it under a name keyed on the version, so without it the first reload serves the old stylesheet. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
@@ -105,7 +105,7 @@ def _shell_call(command: str, *, call_id: str = "c1") -> dict:
|
||||
}
|
||||
|
||||
|
||||
async def _authorise_with(context, calls, *, answers, verdict=interaction.ALLOW):
|
||||
async def _authorise_with(context, calls, *, answers, verdict=interaction.ALLOW, reason=""):
|
||||
"""Run `_authorise` and answer the card it puts up."""
|
||||
generation = generation_service.Generation(chat_id="x", message_id="y")
|
||||
arguments = generation_service._arguments_for(context, calls)
|
||||
@@ -120,7 +120,7 @@ async def _authorise_with(context, calls, *, answers, verdict=interaction.ALLOW)
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
pending = generation.pending
|
||||
pending.resolve(verdict, answers=answers)
|
||||
pending.resolve(verdict, answers=answers, reason=reason)
|
||||
decided, allowed, edited = await task
|
||||
return arguments, decided, allowed, edited, pending
|
||||
|
||||
@@ -223,6 +223,47 @@ async def test_declining_ignores_the_edit(db, user_id, machine):
|
||||
assert arguments[0]["command"] == "pytest"
|
||||
|
||||
|
||||
async def test_a_refusal_with_a_reason_answers_every_call_in_the_round(
|
||||
db, user_id, machine
|
||||
):
|
||||
"""One card covers the round, so one reason answers the round.
|
||||
|
||||
That is the design claim worth pinning: the reader said "not on production"
|
||||
about what they were shown, and what they were shown was both commands. A
|
||||
reason attached to one of them would be the card describing something other
|
||||
than what it asked about.
|
||||
"""
|
||||
context = _context(db, user_id, machine)
|
||||
calls = [_shell_call("pytest", call_id="c1"), _shell_call("make deploy", call_id="c2")]
|
||||
|
||||
_arguments, decided, allowed, _edited, _ = await _authorise_with(
|
||||
context,
|
||||
calls,
|
||||
answers={},
|
||||
verdict=interaction.DENY,
|
||||
reason="not on production",
|
||||
)
|
||||
|
||||
assert allowed == set(), "nothing ran"
|
||||
assert set(decided) == {0, 1}, "and every call was answered without the runner"
|
||||
for outcome in decided.values():
|
||||
assert "not on production" in outcome.content
|
||||
assert outcome.event["error"] == "Declined: not on production"
|
||||
|
||||
|
||||
async def test_a_refusal_with_no_reason_is_unchanged(db, user_id, machine):
|
||||
"""The path everything already took, still taking it."""
|
||||
context = _context(db, user_id, machine)
|
||||
calls = [_shell_call("pytest")]
|
||||
|
||||
_arguments, decided, _allowed, _edited, _ = await _authorise_with(
|
||||
context, calls, answers={}, verdict=interaction.DENY
|
||||
)
|
||||
|
||||
assert decided[0].event["error"] == "Declined."
|
||||
assert "ask what they would prefer" in decided[0].content
|
||||
|
||||
|
||||
async def test_only_the_edited_call_in_a_round_is_changed(db, user_id, machine):
|
||||
"""One card covers the whole round, and the answers are keyed per item."""
|
||||
context = _context(db, user_id, machine)
|
||||
|
||||
@@ -766,7 +766,7 @@ def test_the_endpoint_gathers_every_answer_at_once(client, db, registered, user_
|
||||
chat_id, message_id = _chat_that_can_ask(db, user_id)
|
||||
seen: dict = {}
|
||||
|
||||
def capture(chat, interaction_id, *, verdict="", answers=None):
|
||||
def capture(chat, interaction_id, *, verdict="", answers=None, reason=""):
|
||||
seen["chat"] = chat
|
||||
seen["id"] = interaction_id
|
||||
seen["verdict"] = verdict
|
||||
@@ -811,7 +811,7 @@ def test_the_endpoint_passes_a_verdict_through_untouched(client, db, registered,
|
||||
from lembas.api import chats as chats_api
|
||||
|
||||
original = chats_api.generation_service.answer
|
||||
chats_api.generation_service.answer = lambda c, i, *, verdict="", answers=None: (
|
||||
chats_api.generation_service.answer = lambda c, i, *, verdict="", answers=None, reason="": (
|
||||
seen.update(verdict=verdict, answers=answers) or True
|
||||
)
|
||||
try:
|
||||
@@ -921,8 +921,9 @@ def test_several_ticked_answers_all_come_back(client, db, registered, user_id):
|
||||
from lembas.api import chats as chats_api
|
||||
|
||||
original = chats_api.generation_service.answer
|
||||
chats_api.generation_service.answer = lambda chat, iid, *, verdict="", answers=None: (
|
||||
seen.update(answers=answers) or True
|
||||
chats_api.generation_service.answer = (
|
||||
lambda chat, iid, *, verdict="", answers=None, reason="": seen.update(answers=answers)
|
||||
or True
|
||||
)
|
||||
try:
|
||||
client.post(
|
||||
@@ -944,8 +945,9 @@ def test_something_else_ticked_with_an_empty_box_says_nothing(client, db, regist
|
||||
from lembas.api import chats as chats_api
|
||||
|
||||
original = chats_api.generation_service.answer
|
||||
chats_api.generation_service.answer = lambda chat, iid, *, verdict="", answers=None: (
|
||||
seen.update(answers=answers) or True
|
||||
chats_api.generation_service.answer = (
|
||||
lambda chat, iid, *, verdict="", answers=None, reason="": seen.update(answers=answers)
|
||||
or True
|
||||
)
|
||||
try:
|
||||
client.post(
|
||||
@@ -965,8 +967,9 @@ def test_a_ticked_option_and_something_else_come_back_together(client, db, regis
|
||||
from lembas.api import chats as chats_api
|
||||
|
||||
original = chats_api.generation_service.answer
|
||||
chats_api.generation_service.answer = lambda chat, iid, *, verdict="", answers=None: (
|
||||
seen.update(answers=answers) or True
|
||||
chats_api.generation_service.answer = (
|
||||
lambda chat, iid, *, verdict="", answers=None, reason="": seen.update(answers=answers)
|
||||
or True
|
||||
)
|
||||
try:
|
||||
client.post(
|
||||
@@ -977,3 +980,168 @@ def test_a_ticked_option_and_something_else_come_back_together(client, db, regis
|
||||
chats_api.generation_service.answer = original
|
||||
|
||||
assert seen["answers"] == {"q0": "Tea, or juice"}
|
||||
|
||||
|
||||
# --- Refusing, and saying why --------------------------------------------------
|
||||
# A bare "Don't" tells the model it was refused and nothing else, so it asks what
|
||||
# you would rather -- a whole round spent on something you knew when you pressed
|
||||
# the button. The box is how that round is skipped, and these are the two halves
|
||||
# that have to hold: the reason reaches the model, and it changes what the model
|
||||
# is told to do next.
|
||||
def _approval(key: str = "a0", **kwargs) -> interaction.Item:
|
||||
return interaction.Item(
|
||||
index=0,
|
||||
key=key,
|
||||
kind=interaction.KIND_APPROVAL,
|
||||
tool_name="shell_run",
|
||||
title="Run a command",
|
||||
detail="rm -rf build",
|
||||
**kwargs,
|
||||
)
|
||||
|
||||
|
||||
async def test_a_refusal_can_carry_a_reason():
|
||||
pause = interaction.build("p", [_approval()], timeout=5)
|
||||
pause.resolve(interaction.DENY, reason=" not in that directory, use /srv ")
|
||||
|
||||
reply = await pause._future
|
||||
|
||||
assert reply.permitted is False
|
||||
assert reply.reason == "not in that directory, use /srv", "trimmed, as typed"
|
||||
|
||||
|
||||
async def test_a_plain_refusal_carries_none():
|
||||
"""Which is what keeps the two wordings apart downstream."""
|
||||
pause = interaction.build("p", [_approval()], timeout=5)
|
||||
pause.resolve(interaction.DENY)
|
||||
|
||||
assert (await pause._future).reason == ""
|
||||
|
||||
|
||||
async def test_a_reason_is_bounded():
|
||||
"""It lands in a tool result inside a request that has a window to fit in."""
|
||||
pause = interaction.build("p", [_approval()], timeout=5)
|
||||
pause.resolve(interaction.DENY, reason="x" * (interaction.MAX_REASON_CHARS + 500))
|
||||
|
||||
assert len((await pause._future).reason) == interaction.MAX_REASON_CHARS
|
||||
|
||||
|
||||
def test_the_reason_reaches_the_model_and_redirects_it():
|
||||
"""The instruction has to change, not just the text. "Ask what they would
|
||||
prefer" is right for a model given nothing to go on and exactly wrong for one
|
||||
that has just been told -- it spends a round asking a question whose answer is
|
||||
on the screen above it."""
|
||||
outcome = generation_service._not_allowed(
|
||||
_approval(), interaction.Reply(outcome=interaction.DENY, reason="use /srv instead")
|
||||
)
|
||||
|
||||
assert "use /srv instead" in outcome.content
|
||||
assert "carry on from it" in outcome.content
|
||||
assert "do not ask them to repeat" in outcome.content.lower()
|
||||
assert "ask what they would prefer" not in outcome.content
|
||||
# The half that holds either way: a refusal is not an invitation to find
|
||||
# another route to the same thing.
|
||||
assert "Do not try the same thing another way" in outcome.content
|
||||
|
||||
|
||||
def test_a_refusal_without_a_reason_is_worded_as_it_always_was():
|
||||
outcome = generation_service._not_allowed(
|
||||
_approval(), interaction.Reply(outcome=interaction.DENY)
|
||||
)
|
||||
|
||||
assert "ask what they would prefer" in outcome.content
|
||||
assert outcome.event["error"] == "Declined."
|
||||
|
||||
|
||||
def test_the_reason_is_in_the_transcript_too():
|
||||
"""So somebody scrolling back sees why a step was refused rather than only
|
||||
that it was."""
|
||||
outcome = generation_service._not_allowed(
|
||||
_approval(), interaction.Reply(outcome=interaction.DENY, reason="wrong host")
|
||||
)
|
||||
|
||||
assert outcome.event["error"] == "Declined: wrong host"
|
||||
|
||||
|
||||
def test_an_unanswered_card_is_not_a_refusal_with_a_reason():
|
||||
"""Nobody said anything, so there is nothing to pass on -- and the wording
|
||||
has to stay the one about nobody answering."""
|
||||
outcome = generation_service._not_allowed(
|
||||
_approval(), interaction.Reply(outcome=interaction.EXPIRED)
|
||||
)
|
||||
|
||||
assert "Nobody answered" in outcome.content
|
||||
assert outcome.event["error"] == "Not answered."
|
||||
|
||||
|
||||
def test_the_card_offers_the_box_and_names_the_field(client, db, registered, user_id):
|
||||
pause = interaction.Interruption(id="p1", items=(_approval(),))
|
||||
|
||||
html = _render(pause)
|
||||
|
||||
assert "Give reason" in html
|
||||
assert 'name="reason"' in html
|
||||
# Both halves in the DOM with one hidden, for the reason the edit box gives:
|
||||
# a field created by a click submits nothing when the click handler fails.
|
||||
assert 'x-show="explaining"' in html
|
||||
# Disabled while hidden, so a card refused plainly cannot post an empty
|
||||
# `reason` that would have to be told apart from one somebody cleared.
|
||||
assert ':disabled="!explaining"' in html
|
||||
|
||||
|
||||
def test_the_question_card_offers_no_deny_box():
|
||||
"""It is not a refusal, and it has its own free-text row already."""
|
||||
html = _render(interaction.Interruption(id="p1", items=(_item(),)))
|
||||
|
||||
assert "Give reason" not in html
|
||||
assert 'name="reason"' not in html
|
||||
|
||||
|
||||
def test_the_endpoint_passes_the_reason_on(client, db, registered, user_id):
|
||||
chat_id, _message_id = _chat_that_can_ask(db, user_id)
|
||||
seen: dict = {}
|
||||
|
||||
from lembas.api import chats as chats_api
|
||||
|
||||
original = chats_api.generation_service.answer
|
||||
chats_api.generation_service.answer = (
|
||||
lambda c, i, *, verdict="", answers=None, reason="": seen.update(
|
||||
verdict=verdict, reason=reason
|
||||
)
|
||||
or True
|
||||
)
|
||||
try:
|
||||
client.post(
|
||||
f"/api/chats/{chat_id}/interaction/p1",
|
||||
data={"verdict": "deny", "reason": "not on production"},
|
||||
)
|
||||
finally:
|
||||
chats_api.generation_service.answer = original
|
||||
|
||||
assert seen == {"verdict": "deny", "reason": "not on production"}
|
||||
|
||||
|
||||
def test_a_reason_never_travels_with_a_permission(client, db, registered, user_id):
|
||||
"""Typed into the box, then Allow pressed instead. Reading it there would put
|
||||
the reader's "no, because…" onto a call that went ahead."""
|
||||
chat_id, _message_id = _chat_that_can_ask(db, user_id)
|
||||
seen: dict = {}
|
||||
|
||||
from lembas.api import chats as chats_api
|
||||
|
||||
original = chats_api.generation_service.answer
|
||||
chats_api.generation_service.answer = (
|
||||
lambda c, i, *, verdict="", answers=None, reason="": seen.update(
|
||||
verdict=verdict, reason=reason
|
||||
)
|
||||
or True
|
||||
)
|
||||
try:
|
||||
client.post(
|
||||
f"/api/chats/{chat_id}/interaction/p1",
|
||||
data={"verdict": "allow", "reason": "I changed my mind"},
|
||||
)
|
||||
finally:
|
||||
chats_api.generation_service.answer = original
|
||||
|
||||
assert seen == {"verdict": "allow", "reason": ""}
|
||||
|
||||
Reference in New Issue
Block a user