Houmeres/LLeMbas
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 3 Wiki Activity

PWA, one send/stop button, audio in and out, web search as a tool

Browse Source 
Four pieces of work.

**Installable.** A manifest carrying the instance name, PWA icons rasterised
from the existing mark at design time, a service worker and a themed offline
page. The worker caches the shell only and bails out on /api/, /auth/, /admin/
and anything accepting text/event-stream -- passing a reply stream through a
worker turns it into one delivery at the end, or nothing. It is served from
GET /sw.js rather than the static mount because a worker's scope is the path it
came from.

**Send and Stop are one button.** They were two, and the hidden one was never
hidden: `.btn` is display: inline-flex, which outranks the browser's own
`[hidden] { display: none }`, so Stop sat permanently beside Send. app.css now
forces the attribute to win -- every control toggled with `hidden` depended on
that -- and the composer renders one button carrying both icons, with ui.js
flipping data-composer-action and the type with it.

**Audio.** Speech to text and text to speech against any OpenAI-shaped
/v1/audio/* endpoint: dictate into the composer, have a reply read out.
Instance settings in Admin, per-reader overrides in Settings, with the voice
list discovered from the server where it offers one. Recorded audio is capped
and never written to disk -- it is not an attachment, it has no owner, and
nothing would ever sweep it.

**Web search, as a tool.** This is the tool loop PLAN.md described as the real
work: one reply is now a bounded sequence of requests rather than one. The model
asks, the tool runs, the result goes back and it is asked again, up to three
rounds. Providers are DuckDuckGo (no setup), SearXNG and Firecrawl.

Two decisions worth stating. Tools are only offered to models flagged `tools`,
because an endpoint without support rejects the whole request rather than
ignoring the array -- the same reason images only reach models flagged
`vision`. And tool results are not replayed as context on the next turn, for the
same reasons reasoning is not: the answer already contains what the model made
of them, and replaying stale results into every later request wastes the window
and reliably sends a small model into a search loop. The sources stay visible in
the transcript instead.

Search results are untrusted third-party text and are treated as such: escaped,
and only http/https URLs rendered as links.

338 tests, ruff clean.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Jaroslav Beneš
2026-07-21 17:56:50 +02:00
parent ca3e4fd04f
commit 436226370a
61 changed files with 4481 additions and 116 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/lembas/web/static/js/audio.js
+209
View File
@@ -0,0 +1,209 @@
/*
Dictation and read-aloud.
Both halves are progressive: without this file the composer and the message
bubbles still work, they simply have two buttons that do nothing. Neither
feature's markup is rendered at all unless an administrator has configured an
endpoint for it, so that state is rare rather than normal.
*/
(function () {
"use strict";
function notify(message, kind) {
if (window.lembas && window.lembas.notify) {
window.lembas.notify(message, { kind: kind || "info" });
}
}
/* --- Dictation ---------------------------------------------------------
MediaRecorder writes whatever container the browser prefers -- webm/opus
almost everywhere, mp4 on Safari. The file is passed upstream with the
type the browser reported rather than being converted here: whisper.cpp
and friends decode through ffmpeg and take all of them, and converting in
the browser would mean shipping an encoder. */
var recorder = null;
var chunks = [];
var micButton = null;
function setMicState(button, state) {
if (!button) return;
button.dataset.micState = state;
button.disabled = state === "working";
button.setAttribute(
"aria-label",
state === "recording" ? "Stop recording" : "Dictate a message"
);
button.title = button.getAttribute("aria-label");
}
function composerInput() {
return document.querySelector("[data-composer-input]");
}
function insertTranscript(text) {
var input = composerInput();
if (!input || !text) return;
// Appended rather than replacing: dictation is usually finishing a thought
// that was already half typed.
var existing = input.value.trim();
input.value = existing ? existing + " " + text : text;
if (window.lembas && window.lembas.autosize) window.lembas.autosize(input);
input.focus();
input.selectionStart = input.selectionEnd = input.value.length;
}
function upload(blob, button) {
var body = new FormData();
// The extension only has to be something the server can name the part;
// the endpoint sniffs the container itself.
var extension = (blob.type.indexOf("mp4") !== -1) ? "mp4" : "webm";
body.append("file", blob, "dictation." + extension);
setMicState(button, "working");
fetch("/api/audio/transcribe", {
method: "POST",
body: body,
credentials: "same-origin",
})
.then(function (response) {
if (!response.ok) {
return response.json()
.catch(function () { return {}; })
.then(function (payload) {
throw new Error(payload.detail || "Transcription failed.");
});
}
return response.text();
})
.then(function (text) {
setMicState(button, "idle");
if (!text.trim()) {
notify("Nothing was heard in that recording.", "info");
return;
}
insertTranscript(text.trim());
})
.catch(function (error) {
setMicState(button, "idle");
notify(error.message || "Transcription failed.", "error");
});
}
function startRecording(button) {
/* getUserMedia is undefined on plain http, which a self-hosted install on
a LAN address often is. Saying so beats a button that silently does
nothing -- the fix is not something the page can apply for them. */
if (!navigator.mediaDevices || !navigator.mediaDevices.getUserMedia ||
typeof MediaRecorder === "undefined") {
notify(
"The microphone needs HTTPS or localhost. This page is served over " +
"plain HTTP, so the browser will not grant it.",
"error"
);
return;
}
navigator.mediaDevices.getUserMedia({ audio: true }).then(function (stream) {
chunks = [];
recorder = new MediaRecorder(stream);
micButton = button;
recorder.addEventListener("dataavailable", function (event) {
if (event.data && event.data.size) chunks.push(event.data);
});
recorder.addEventListener("stop", function () {
// Release the microphone immediately: leaving the track live keeps the
// browser's recording indicator on long after anyone is talking.
stream.getTracks().forEach(function (track) { track.stop(); });
var blob = new Blob(chunks, { type: recorder.mimeType || "audio/webm" });
recorder = null;
if (blob.size) upload(blob, button); else setMicState(button, "idle");
});
recorder.start();
setMicState(button, "recording");
}).catch(function () {
notify("The microphone could not be opened. Permission may be blocked.", "error");
});
}
function stopRecording() {
if (recorder && recorder.state !== "inactive") recorder.stop();
}
/* --- Reading a reply aloud ---------------------------------------------
One <audio> element for the whole page. Two replies talking over each
other is never what was wanted, and a shared element makes that
impossible rather than merely unlikely. */
var player = null;
var speaking = null;
function audioPlayer() {
if (!player) {
player = new Audio();
player.addEventListener("ended", function () { markSpeaking(null); });
player.addEventListener("error", function () {
if (speaking) notify("That reply could not be read out.", "error");
markSpeaking(null);
});
}
return player;
}
function markSpeaking(button) {
document.querySelectorAll("[data-speak]").forEach(function (el) {
el.classList.toggle("is-speaking", el === button);
});
speaking = button;
}
function speak(button) {
var element = audioPlayer();
if (speaking === button) {
element.pause();
markSpeaking(null);
return;
}
element.pause();
element.src = button.dataset.speak;
markSpeaking(button);
element.play().catch(function () {
/* Autoplay policies reject a play() the reader did not ask for. That is
the browser working as intended, so it is not reported as an error. */
markSpeaking(null);
});
}
/* --- Wiring ------------------------------------------------------------ */
document.addEventListener("click", function (event) {
var mic = event.target.closest("[data-mic]");
if (mic) {
event.preventDefault();
if (mic.dataset.micState === "recording") stopRecording();
else if (mic.dataset.micState === "idle") startRecording(mic);
return;
}
var speaker = event.target.closest("[data-speak]");
if (speaker) {
event.preventDefault();
speak(speaker);
}
});
/* A reply that has just finished streaming carries data-speak-auto, set only
on that one frame. Any swap can bring it in, so this watches them all and
clears the attribute after acting -- a later swap of the same bubble must
not start it again. */
function playArrivals() {
document.querySelectorAll("[data-speak-auto]").forEach(function (button) {
button.removeAttribute("data-speak-auto");
speak(button);
});
}
document.addEventListener("DOMContentLoaded", playArrivals);
if (document.body) {
document.body.addEventListener("htmx:afterSettle", playArrivals);
}
})();