Voice (dictate, read aloud) and Draw a picture over /v1, with the app's models scope

The server kept voice and images on /v1 (models scope, which the app's token
already asks for); /me's services and scopes decide what is offered. Rust
sends the recording to /v1/audio/transcriptions, reads /v1/audio/speech back
as a data: URL (CSP media-src data:), and for /v1/images/generations takes
only the id of each of the person's own files from the answer and fetches
/v1/files/<id>/content from the app's own server. The microphone is allowed
only within seconds of the page's own Dictate (voice_arm); WebKitGTK's
media stream is switched on through its settings getter.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ui/js/voice.js

@@ -0,0 +1,180 @@
+// Voice: a message dictated in the composer, and a reply read aloud. Both are the instance's own
+// speech services on the OpenAI-compatible routes, which the app's token reaches through its
+// `models` scope (src-tauri/src/server.rs):
+//
+//   POST /v1/audio/transcriptions   the recording -> { text }       (Rust: `transcribe`)
+//   POST /v1/audio/speech           { input } -> the audio           (Rust: `speak`, a data: URL)
+//
+// Each is offered only where `GET /me` says the instance has it for this account (`services.stt`,
+// `services.tts`) and the token holds `models` (`scopes`), so no button is there that can only fail.
+// The microphone is asked for only when Dictate is pressed: Rust allows it for that moment alone.
+
+import { $, h, icon, ICONS } from "./dom.js"
+import { invoke, why } from "./bridge.js"
+
+let me = null
+
+const hasModels = () => !Array.isArray(me?.scopes) || me.scopes.includes("models")
+/** Whether a message can be dictated here. */
+export const canDictate = () => !!me?.services?.stt && hasModels() && me?.permissions?.["audio.transcribe"] !== false
+/** Whether a reply can be read aloud here. */
+export const canSpeak = () => !!me?.services?.tts && hasModels() && me?.permissions?.["audio.listen"] !== false
+
+function say(text) {
+  const box = $("thread-note")
+  if (!box) return
+  box.textContent = text
+  box.hidden = !text
+}
+
+// --- Dictate --------------------------------------------------------------------------------------
+
+let recorder = null // the recording in progress
+let busy = false // a recording being turned into words
+
+function drawMic() {
+  const b = $("dictate")
+  if (!b) return
+  b.hidden = !canDictate()
+  const on = !!recorder
+  b.classList.toggle("is-recording", on)
+  b.classList.toggle("is-busy", busy)
+  b.disabled = busy
+  b.setAttribute("aria-pressed", String(on))
+  const label = busy ? t("Writing down…") : on ? t("Stop dictating") : t("Dictate a message")
+  b.setAttribute("aria-label", label)
+  b.title = label
+}
+
+async function start() {
+  say("")
+  let stream
+  try {
+    await invoke("voice_arm")
+    stream = await navigator.mediaDevices.getUserMedia({ audio: true })
+  } catch (err) {
+    say(t("The microphone could not be opened: %(why)s", { why: err?.message || why(err) }))
+    return
+  }
+  const type = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/webm"].find((x) => window.MediaRecorder?.isTypeSupported?.(x))
+  let rec
+  try {
+    rec = new MediaRecorder(stream, type ? { mimeType: type } : undefined)
+  } catch (err) {
+    stream.getTracks().forEach((x) => x.stop())
+    say(t("The microphone could not be opened: %(why)s", { why: err?.message || why(err) }))
+    return
+  }
+  const chunks = []
+  rec.addEventListener("dataavailable", (e) => e.data.size && chunks.push(e.data))
+  rec.addEventListener("stop", async () => {
+    stream.getTracks().forEach((x) => x.stop())
+    recorder = null
+    const blob = new Blob(chunks, { type: rec.mimeType || type || "audio/webm" })
+    if (!blob.size) return drawMic()
+    busy = true
+    drawMic()
+    try {
+      const text = await invoke("transcribe", new Uint8Array(await blob.arrayBuffer()), { headers: { "x-type": blob.type } })
+      intoComposer(text)
+    } catch (err) {
+      say(why(err))
+    }
+    busy = false
+    drawMic()
+  })
+  recorder = rec
+  rec.start()
+  drawMic()
+}
+
+/** The words heard, where the cursor is in the message box, a space either side where needed. */
+function intoComposer(text) {
+  text = String(text || "").trim()
+  if (!text) return
+  const box = $("composer-text")
+  const at = box.selectionStart ?? box.value.length
+  const end = box.selectionEnd ?? at
+  const before = box.value.slice(0, at)
+  const after = box.value.slice(end)
+  const words = `${before && !/\s$/.test(before) ? " " : ""}${text}${after && !/^\s/.test(after) ? " " : ""}`
+  box.value = before + words + after
+  box.selectionStart = box.selectionEnd = before.length + words.length
+  box.dispatchEvent(new Event("input"))
+  box.focus()
+}
+
+/** Stop a recording (a chat closed, the window hidden): what was said is still written down. */
+export function stopDictating() {
+  if (recorder?.state === "recording") recorder.stop()
+}
+
+// --- Read aloud -----------------------------------------------------------------------------------
+
+let playing = null // { id, audio, button }
+
+/** A reply's words without the markdown around them, and without its code. */
+export function plain(markdown) {
+  return String(markdown || "")
+    .replace(/```[\s\S]*?```/g, " ")
+    .replace(/!?\[([^\]]*)]\([^)]*\)/g, "$1")
+    .replace(/[*_`#>|~]/g, "")
+    .replace(/\s+/g, " ")
+    .trim()
+}
+
+function stopReading() {
+  if (!playing) return
+  playing.audio?.pause()
+  playing.button.setAttribute("aria-pressed", "false")
+  playing.button.classList.remove("is-busy")
+  playing = null
+}
+
+/** The Read aloud button of a reply (`id`), reading `text()` when pressed; null where it cannot. */
+export function readAloudButton(id, text) {
+  if (!canSpeak()) return null
+  const label = t("Read this aloud")
+  const b = h("button", { type: "button", class: "msg__act msg__speak", "aria-label": label, title: label, "aria-pressed": "false" }, icon(ICONS.speaker))
+  b.addEventListener("click", async () => {
+    if (playing?.id === id) return stopReading()
+    stopReading()
+    const words = plain(text())
+    if (!words) return
+    const mine = { id, audio: null, button: b }
+    playing = mine
+    b.setAttribute("aria-pressed", "true")
+    b.classList.add("is-busy")
+    try {
+      const url = await invoke("speak", { text: words })
+      if (playing !== mine) return
+      // Only audio the app's own server sent, as Rust read it: a data: URL of an audio type.
+      if (!/^data:audio\/[a-z0-9.+-]+;base64,/i.test(url)) throw new Error(t("The server's answer was not audio."))
+      mine.audio = new Audio(url)
+      b.classList.remove("is-busy")
+      mine.audio.addEventListener("ended", () => playing === mine && stopReading())
+      await mine.audio.play()
+    } catch (err) {
+      if (playing === mine) stopReading()
+      say(why(err?.message ? { detail: err.message } : err))
+    }
+  })
+  return b
+}
+
+/** What the account may use, from `GET /me`: the composer's Dictate button is drawn by it. */
+export function voiceFor(account) {
+  me = account
+  drawMic()
+}
+
+export function initVoice() {
+  const b = $("dictate")
+  b.replaceChildren(icon(ICONS.mic))
+  b.addEventListener("click", () => {
+    if (busy) return
+    if (recorder) recorder.stop()
+    else start()
+  })
+  drawMic()
+}