Wake the model when a background job finishes

The other half of background execution: a job that finishes while nobody is
looking prompts the model back with its result, rather than sitting unread until
the model happens to run again.

The vehicle is the queue, because it is the only wiring that already delivers a
turn into or after a reply. A per-job poller notices completion and calls
jobs.wake. If a reply is being written the completion is left queued for that
reply's _inject/_drain; if the chat is idle a fresh reply is started to answer
it -- the send_queued_now move. All of it under a per-chat lock with no await
between the running-check and ensure, so two jobs finishing at once cannot each
spin up a generation: the second sees the first's reply already live and leaves
its completion for it. That is the invariant the queue exists to hold, reached
from outside a request for the first time.

The completion is a user-role turn whose content names itself a machine event --
"A background job you started has finished" -- not a bare person turn. _inject
sends a queued turn verbatim, so the framing cannot live there; it lives in the
words, the way execute_plan quotes the plan, and a tool.background fragment tells
the model these arrive and are a machine event rather than the person speaking.

The poller reconnects a fresh connection each tick rather than holding one open
-- holding one is the exact live-connection state the whole ssh.py/base.py design
forbids, and poll is self-healing besides. Bounded by background_max_jobs and a
six-hour ceiling, after which the remote job may keep running but we stop
watching it.

A Job table, and here the terminal/generation "lost on restart" precedent does
NOT transfer: those are seconds long with a human watching, a background job is
hours long with nobody watching -- the one case a restart forgetting it would
silently break the feature's whole promise. So the row lets a lifespan startup
hook rehydrate the watcher and wake as if nothing happened. Cancelling a watcher
never stops the detached remote job; it runs on and is picked back up.

Tested end to end against a real local shell: launch a detached command, poll it
to completion through a watcher, and assert the model was woken with the exit
code and output -- plus the lock proving two simultaneous completions start one
reply, not two.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

tests/test_agent_jobs.py

@@ -281,3 +281,134 @@ def test_background_param_appears_only_when_enabled(tmp_path):
     off = shell(_agent(tmp_path, background=False)).parameters["properties"]
     assert "background" in on
     assert "background" not in off
+
+
+# --- Waking the model when a job finishes --------------------------------------
+def _chat(db, user_id) -> str:
+    from lembas.db.models import Chat
+
+    chat = Chat(user_id=user_id, model_id="m")
+    db.add(chat)
+    db.commit()
+    return chat.id
+
+
+def test_the_completion_names_itself_a_machine_event():
+    text = jobs._completion_text("abc123abc123", "apt-get install -y x", "done", 0, "ok\n")
+    assert "machine event" in text
+    assert "[job abc123abc123]" in text
+    assert "apt-get install" in text
+    assert "finished successfully" in text.lower()
+
+
+def test_a_nonzero_completion_reports_the_code():
+    text = jobs._completion_text("j", "build", "done", 2, "")
+    assert "exited 2" in text
+
+
+def test_backticks_in_output_cannot_close_the_fence():
+    text = jobs._completion_text("j", "c", "done", 0, "see ```code``` here")
+    assert "```code```" not in text
+
+
+async def test_an_idle_chat_gets_a_fresh_reply(db, user_id, registered, monkeypatch):
+    from lembas.services import generation as generation_service
+
+    chat_id = _chat(db, user_id)
+    monkeypatch.setattr(generation_service, "running_for", lambda _c: None)
+    started: list[str] = []
+    monkeypatch.setattr(generation_service, "ensure", lambda c, m: started.append(m))
+
+    await jobs.wake(chat_id, "abc123abc123", "sleep 1", "done", 0, "hi")
+
+    from lembas.db.models import Message
+
+    db.expire_all()
+    msgs = db.query(Message).filter(Message.chat_id == chat_id).all()
+    users = [m for m in msgs if m.role == "user"]
+    assert len(users) == 1
+    assert users[0].queued is False, "an idle chat's completion is delivered, not queued"
+    assert any(m.role == "assistant" and not m.complete for m in msgs)
+    assert len(started) == 1, "a reply was started"
+
+
+async def test_a_busy_chat_gets_a_queued_turn_and_no_new_reply(
+    db, user_id, registered, monkeypatch
+):
+    from lembas.services import generation as generation_service
+
+    chat_id = _chat(db, user_id)
+    monkeypatch.setattr(generation_service, "running_for", lambda _c: object())
+    started: list[str] = []
+    monkeypatch.setattr(generation_service, "ensure", lambda c, m: started.append(m))
+
+    await jobs.wake(chat_id, "abc123abc123", "sleep 1", "done", 0, "hi")
+
+    from lembas.db.models import Message
+
+    db.expire_all()
+    users = [
+        m for m in db.query(Message).filter(Message.chat_id == chat_id).all() if m.role == "user"
+    ]
+    assert len(users) == 1
+    assert users[0].queued is True, "a running reply's _inject/_drain will deliver it"
+    assert started == [], "no second reply for a chat already writing one"
+
+
+async def test_two_jobs_finishing_at_once_start_one_reply(db, user_id, registered, monkeypatch):
+    """The per-chat lock. Without it, both wakes would each start a generation."""
+    from lembas.services import generation as generation_service
+
+    chat_id = _chat(db, user_id)
+    live = {"running": False}
+    monkeypatch.setattr(
+        generation_service, "running_for", lambda _c: object() if live["running"] else None
+    )
+    started: list[str] = []
+
+    def _ensure(c, m):
+        live["running"] = True  # what running_for will now see
+        started.append(m)
+
+    monkeypatch.setattr(generation_service, "ensure", _ensure)
+
+    await asyncio.gather(
+        jobs.wake(chat_id, "aaaaaaaaaaaa", "a", "done", 0, "x"),
+        jobs.wake(chat_id, "bbbbbbbbbbbb", "b", "done", 0, "y"),
+    )
+
+    assert len(started) == 1, "the lock made the second wake see the first's reply"
+
+
+# --- The watcher, end to end ---------------------------------------------------
+async def test_the_watcher_end_to_end(db, user_id, registered, monkeypatch, tmp_path):
+    from lembas.services.agent import ssh as ssh_service
+
+    chat_id = _chat(db, user_id)
+
+    class _Local(LocalExecutor):
+        def __init__(self, spec, project_dir):
+            super().__init__(str(tmp_path))
+
+    monkeypatch.setattr(ssh_service, "SshExecutor", _Local)
+    monkeypatch.setattr(jobs, "_poll_interval", lambda _e: 0.1)
+    woken: list = []
+
+    async def _wake(chat, job_id, command, status, code, body):
+        woken.append((status, code, body))
+
+    monkeypatch.setattr(jobs, "wake", _wake)
+
+    agent = _agent(tmp_path, chat_id=chat_id)
+    agent.executor = lambda: _Local(None, str(tmp_path))
+    job = await jobs.launch(agent, "sleep 0.5; echo finished")
+    jobs.start_watch(agent, job)
+
+    for _ in range(50):
+        if woken:
+            break
+        await asyncio.sleep(0.1)
+    assert woken, "the watcher never woke the model"
+    status, code, body = woken[0]
+    assert status == "done"
+    assert "finished" in body