Voice (dictate, read aloud) and Draw a picture over /v1, with the app's models scope

The server kept voice and images on /v1 (models scope, which the app's token
already asks for); /me's services and scopes decide what is offered. Rust
sends the recording to /v1/audio/transcriptions, reads /v1/audio/speech back
as a data: URL (CSP media-src data:), and for /v1/images/generations takes
only the id of each of the person's own files from the answer and fetches
/v1/files/<id>/content from the app's own server. The microphone is allowed
only within seconds of the page's own Dictate (voice_arm); WebKitGTK's
media stream is switched on through its settings getter.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

ui/js/chat.js

@@ -22,6 +22,7 @@ import { unarchive } from "./archived.js"
 import { isProgress, progressLine, progressTicks } from "./progress.js"
 import { aimTerminal, terminalButton, terminalOpen, toggleTerminal } from "./terminal.js"
 import { scopePopover } from "./scope.js"
+import { readAloudButton } from "./voice.js"
 
 const MODES = ["manual", "edit", "auto", "plan"]
 
@@ -292,6 +293,7 @@ function assistantMessage(m, chat) {
         "div",
         { class: "msg__actions" },
         action(ICONS.copy, t("Copy"), () => copyText(replyText(m))),
+        readAloudButton(m.id, () => replyText(m)),
         // Only the last reply: CSS shows it there and nowhere else (app.css, .msg__regen).
         action(ICONS.refresh, t("Regenerate reply"), () => regenerate(m), "msg__regen"),
       ),

ui/js/dom.js

@@ -101,6 +101,9 @@ export const ICONS = {
   bell: ["M8 1.5a4.5 4.5 0 014.5 4.5v3l1.5 2.5H2L3.5 9V6A4.5 4.5 0 018 1.5zM8 3a3 3 0 00-3 3v3.42l-.35.58h6.7L11 9.42V6a3 3 0 00-3-3z", "M6.25 12.5h3.5a1.75 1.75 0 01-3.5 0z"],
   archive: ["M1.5 2.5h13v3.5h-13zM3 4v.5h10V4z", "M2.5 6.5H4V12h8V6.5h1.5v7h-11z", "M6 7.5h4V9H6z"],
   contrast: ["M8 1.5a6.5 6.5 0 110 13 6.5 6.5 0 010-13zM8 3a5 5 0 100 10A5 5 0 008 3z", "M8 3a5 5 0 010 10z"],
+  mic: ["M8 1.5a2.5 2.5 0 012.5 2.5v4a2.5 2.5 0 01-5 0V4A2.5 2.5 0 018 1.5zM8 3a1 1 0 00-1 1v4a1 1 0 002 0V4a1 1 0 00-1-1z", "M3.5 7.5H5a3 3 0 006 0h1.5a4.5 4.5 0 01-3.75 4.44V14h-1.5v-2.06A4.5 4.5 0 013.5 7.5z"],
+  speaker: ["M2 6h2.5L8 3v10L4.5 10H2z", "M10.2 5.2a4 4 0 010 5.6l-1.06-1.06a2.5 2.5 0 000-3.48z", "M12.3 3.1a7 7 0 010 9.8l-1.06-1.06a5.5 5.5 0 000-7.68z"],
+  picture: ["M1.5 2.5h13v11h-13zM3 4v6.4l2.75-2.75 2.5 2.5 1.75-1.75L13 11.4V4z", "M10.5 5a1.25 1.25 0 110 2.5 1.25 1.25 0 010-2.5z"],
   info: ["M8 1.5a6.5 6.5 0 110 13 6.5 6.5 0 010-13zM8 3a5 5 0 100 10A5 5 0 008 3z", "M7.25 7h1.5v4.5h-1.5zM7.25 4.5h1.5V6h-1.5z"],
 }
 

ui/js/draw.js

@@ -0,0 +1,86 @@
+// Draw a picture: the instance's own image generation (`POST /v1/images/generations`, its
+// workflow and default checkpoint), from Ctrl+K. Offered only where `GET /me` says the account may
+// (`services.images`) and the token holds `models`.
+//
+// The pictures are shown only as Rust read them: each one of the person's own files on this
+// server (`/v1/files/<id>/content`, fetched by id from the app's own server, never from an
+// address the answer named), handed over as a `data:` URL of an image type -- checked again here
+// before it becomes an <img>.
+
+import { h, icon, ICONS } from "./dom.js"
+import { invoke, why } from "./bridge.js"
+
+const PICTURE = /^data:image\/(png|jpeg|webp|gif);base64,[A-Za-z0-9+/=]+$/
+
+/** Whether pictures can be drawn here, for the account `me` (`GET /me`). */
+export function canDraw(me) {
+  const models = !Array.isArray(me?.scopes) || me.scopes.includes("models")
+  return !!me?.services?.images && models && me?.permissions?.["tools.image"] !== false
+}
+
+/** The dialog: a prompt, a shape, Draw, and the pictures under it. */
+export function openDraw() {
+  const prompt = h("textarea", { class: "textarea draw__prompt", rows: "3", placeholder: t("What to draw…"), "aria-label": t("What to draw…") })
+  const size = h(
+    "select",
+    { class: "draw__size", "aria-label": t("Shape") },
+    h("option", { value: "1024x1024", text: t("Square") }),
+    h("option", { value: "1344x768", text: t("Wide") }),
+    h("option", { value: "768x1344", text: t("Tall") }),
+  )
+  const status = h("p", { class: "status", role: "status", hidden: true })
+  const out = h("div", { class: "draw__out" })
+  const go = h("button", { type: "submit", class: "btn btn--primary", text: t("Draw") })
+  const form = h(
+    "form",
+    { class: "dialog__body draw", method: "dialog" },
+    h("h2", { class: "dialog__title" }, icon(ICONS.picture), h("span", { text: t("Draw a picture") })),
+    prompt,
+    h("div", { class: "draw__bar" }, size, h("button", { type: "button", class: "btn", text: t("Close"), onclick: () => close() }), go),
+    status,
+    out,
+  )
+  const dlg = h("dialog", { class: "dialog dialog--wide" }, form)
+  let asked = 0
+  const close = () => {
+    asked++
+    dlg.close()
+    dlg.remove()
+  }
+  const said = (text, error = false) => {
+    status.hidden = !text
+    status.textContent = text
+    status.className = `status${error ? " is-error" : ""}`
+    status.setAttribute("role", error ? "alert" : "status")
+  }
+  form.addEventListener("submit", async (e) => {
+    e.preventDefault()
+    const words = prompt.value.trim()
+    if (!words) return prompt.focus()
+    const mine = ++asked
+    go.disabled = true
+    said(t("Drawing… a picture can take a minute or two."))
+    try {
+      const urls = await invoke("draw", { prompt: words, size: size.value })
+      if (mine !== asked) return
+      const pictures = (urls || []).filter((u) => PICTURE.test(u))
+      if (!pictures.length) throw { detail: t("The server's answer had no picture in it.") }
+      out.prepend(...pictures.map((src) => h("figure", { class: "draw__pic" }, h("img", { src, alt: words }), h("figcaption", { text: words }))))
+      said("")
+    } catch (err) {
+      if (mine === asked) said(why(err), true)
+    } finally {
+      if (mine === asked) go.disabled = false
+    }
+  })
+  prompt.addEventListener("keydown", (e) => {
+    if (e.key === "Enter" && (e.ctrlKey || e.metaKey)) form.requestSubmit()
+  })
+  dlg.addEventListener("cancel", (e) => {
+    e.preventDefault()
+    close()
+  })
+  document.body.append(dlg)
+  dlg.showModal()
+  prompt.focus()
+}

ui/js/main.js

@@ -22,6 +22,8 @@ import { initAgent, link } from "./agent.js"
 import { initScheduled, showScheduled, scheduledCommands } from "./scheduled.js"
 import { initArchived, showArchived } from "./archived.js"
 import { showMemory } from "./memory.js"
+import { initVoice, voiceFor } from "./voice.js"
+import { canDraw, openDraw } from "./draw.js"
 import { closePopover } from "./popup.js"
 import { noteSubagentsCap } from "./models.js"
 
@@ -94,6 +96,7 @@ async function startApp() {
   try {
     me = await get("/me")
     noteMe(me)
+    voiceFor(me)
   } catch (err) {
     $("thread-note").textContent = why(err)
     $("thread-note").hidden = false
@@ -233,6 +236,7 @@ function commands() {
     { label: t("Memory"), run: () => showMemoryPage() },
     { label: t("Settings"), run: () => showSettings(true) },
   ]
+  if (canDraw(me)) out.push({ label: t("Draw a picture"), run: openDraw })
   if (openChatId()) out.push({ label: panelOpen() ? t("Close the activity panel") : t("Open the activity panel"), run: togglePanel })
   if (terminalAvailable()) out.push({ label: terminalOpen() ? t("Close terminal") : t("Terminal"), run: toggleTerminal })
   if (status?.linked) out.push({ label: status.agent_wanted ? t("Stop the agent") : t("Start the agent"), run: () => invoke("set_agent", { on: !status.agent_wanted }) })
@@ -280,6 +284,7 @@ async function boot() {
       $("thread-note").hidden = !text
     },
   })
+  initVoice()
   initChat({
     status: () => status,
     thisDevices,

ui/js/voice.js

@@ -0,0 +1,180 @@
+// Voice: a message dictated in the composer, and a reply read aloud. Both are the instance's own
+// speech services on the OpenAI-compatible routes, which the app's token reaches through its
+// `models` scope (src-tauri/src/server.rs):
+//
+//   POST /v1/audio/transcriptions   the recording -> { text }       (Rust: `transcribe`)
+//   POST /v1/audio/speech           { input } -> the audio           (Rust: `speak`, a data: URL)
+//
+// Each is offered only where `GET /me` says the instance has it for this account (`services.stt`,
+// `services.tts`) and the token holds `models` (`scopes`), so no button is there that can only fail.
+// The microphone is asked for only when Dictate is pressed: Rust allows it for that moment alone.
+
+import { $, h, icon, ICONS } from "./dom.js"
+import { invoke, why } from "./bridge.js"
+
+let me = null
+
+const hasModels = () => !Array.isArray(me?.scopes) || me.scopes.includes("models")
+/** Whether a message can be dictated here. */
+export const canDictate = () => !!me?.services?.stt && hasModels() && me?.permissions?.["audio.transcribe"] !== false
+/** Whether a reply can be read aloud here. */
+export const canSpeak = () => !!me?.services?.tts && hasModels() && me?.permissions?.["audio.listen"] !== false
+
+function say(text) {
+  const box = $("thread-note")
+  if (!box) return
+  box.textContent = text
+  box.hidden = !text
+}
+
+// --- Dictate --------------------------------------------------------------------------------------
+
+let recorder = null // the recording in progress
+let busy = false // a recording being turned into words
+
+function drawMic() {
+  const b = $("dictate")
+  if (!b) return
+  b.hidden = !canDictate()
+  const on = !!recorder
+  b.classList.toggle("is-recording", on)
+  b.classList.toggle("is-busy", busy)
+  b.disabled = busy
+  b.setAttribute("aria-pressed", String(on))
+  const label = busy ? t("Writing down…") : on ? t("Stop dictating") : t("Dictate a message")
+  b.setAttribute("aria-label", label)
+  b.title = label
+}
+
+async function start() {
+  say("")
+  let stream
+  try {
+    await invoke("voice_arm")
+    stream = await navigator.mediaDevices.getUserMedia({ audio: true })
+  } catch (err) {
+    say(t("The microphone could not be opened: %(why)s", { why: err?.message || why(err) }))
+    return
+  }
+  const type = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/webm"].find((x) => window.MediaRecorder?.isTypeSupported?.(x))
+  let rec
+  try {
+    rec = new MediaRecorder(stream, type ? { mimeType: type } : undefined)
+  } catch (err) {
+    stream.getTracks().forEach((x) => x.stop())
+    say(t("The microphone could not be opened: %(why)s", { why: err?.message || why(err) }))
+    return
+  }
+  const chunks = []
+  rec.addEventListener("dataavailable", (e) => e.data.size && chunks.push(e.data))
+  rec.addEventListener("stop", async () => {
+    stream.getTracks().forEach((x) => x.stop())
+    recorder = null
+    const blob = new Blob(chunks, { type: rec.mimeType || type || "audio/webm" })
+    if (!blob.size) return drawMic()
+    busy = true
+    drawMic()
+    try {
+      const text = await invoke("transcribe", new Uint8Array(await blob.arrayBuffer()), { headers: { "x-type": blob.type } })
+      intoComposer(text)
+    } catch (err) {
+      say(why(err))
+    }
+    busy = false
+    drawMic()
+  })
+  recorder = rec
+  rec.start()
+  drawMic()
+}
+
+/** The words heard, where the cursor is in the message box, a space either side where needed. */
+function intoComposer(text) {
+  text = String(text || "").trim()
+  if (!text) return
+  const box = $("composer-text")
+  const at = box.selectionStart ?? box.value.length
+  const end = box.selectionEnd ?? at
+  const before = box.value.slice(0, at)
+  const after = box.value.slice(end)
+  const words = `${before && !/\s$/.test(before) ? " " : ""}${text}${after && !/^\s/.test(after) ? " " : ""}`
+  box.value = before + words + after
+  box.selectionStart = box.selectionEnd = before.length + words.length
+  box.dispatchEvent(new Event("input"))
+  box.focus()
+}
+
+/** Stop a recording (a chat closed, the window hidden): what was said is still written down. */
+export function stopDictating() {
+  if (recorder?.state === "recording") recorder.stop()
+}
+
+// --- Read aloud -----------------------------------------------------------------------------------
+
+let playing = null // { id, audio, button }
+
+/** A reply's words without the markdown around them, and without its code. */
+export function plain(markdown) {
+  return String(markdown || "")
+    .replace(/```[\s\S]*?```/g, " ")
+    .replace(/!?\[([^\]]*)]\([^)]*\)/g, "$1")
+    .replace(/[*_`#>|~]/g, "")
+    .replace(/\s+/g, " ")
+    .trim()
+}
+
+function stopReading() {
+  if (!playing) return
+  playing.audio?.pause()
+  playing.button.setAttribute("aria-pressed", "false")
+  playing.button.classList.remove("is-busy")
+  playing = null
+}
+
+/** The Read aloud button of a reply (`id`), reading `text()` when pressed; null where it cannot. */
+export function readAloudButton(id, text) {
+  if (!canSpeak()) return null
+  const label = t("Read this aloud")
+  const b = h("button", { type: "button", class: "msg__act msg__speak", "aria-label": label, title: label, "aria-pressed": "false" }, icon(ICONS.speaker))
+  b.addEventListener("click", async () => {
+    if (playing?.id === id) return stopReading()
+    stopReading()
+    const words = plain(text())
+    if (!words) return
+    const mine = { id, audio: null, button: b }
+    playing = mine
+    b.setAttribute("aria-pressed", "true")
+    b.classList.add("is-busy")
+    try {
+      const url = await invoke("speak", { text: words })
+      if (playing !== mine) return
+      // Only audio the app's own server sent, as Rust read it: a data: URL of an audio type.
+      if (!/^data:audio\/[a-z0-9.+-]+;base64,/i.test(url)) throw new Error(t("The server's answer was not audio."))
+      mine.audio = new Audio(url)
+      b.classList.remove("is-busy")
+      mine.audio.addEventListener("ended", () => playing === mine && stopReading())
+      await mine.audio.play()
+    } catch (err) {
+      if (playing === mine) stopReading()
+      say(why(err?.message ? { detail: err.message } : err))
+    }
+  })
+  return b
+}
+
+/** What the account may use, from `GET /me`: the composer's Dictate button is drawn by it. */
+export function voiceFor(account) {
+  me = account
+  drawMic()
+}
+
+export function initVoice() {
+  const b = $("dictate")
+  b.replaceChildren(icon(ICONS.mic))
+  b.addEventListener("click", () => {
+    if (busy) return
+    if (recorder) recorder.stop()
+    else start()
+  })
+  drawMic()
+}