LLeMbas-Desktop/ui/js/voice.js

// Voice: a message dictated in the composer, and a reply read aloud. Both are the instance's own
// speech services on the OpenAI-compatible routes, which the app's token reaches through its
// `models` scope (src-tauri/src/server.rs):
//
//   POST /v1/audio/transcriptions   the recording -> { text }       (Rust: `transcribe`)
//   POST /v1/audio/speech           { input } -> the audio           (Rust: `speak`, a data: URL)
//
// Each is offered only where `GET /me` says the instance has it for this account (`services.stt`,
// `services.tts`) and the token holds `models` (`scopes`), so no button is there that can only fail.
// The microphone is asked for only when Dictate is pressed: Rust allows it for that moment alone.

import { $, h, icon, ICONS } from "./dom.js"
import { invoke, why } from "./bridge.js"

let me = null

const hasModels = () => !Array.isArray(me?.scopes) || me.scopes.includes("models")
/** Whether a message can be dictated here. */
export const canDictate = () => !!me?.services?.stt && hasModels() && me?.permissions?.["audio.transcribe"] !== false
/** Whether a reply can be read aloud here. */
export const canSpeak = () => !!me?.services?.tts && hasModels() && me?.permissions?.["audio.listen"] !== false

function say(text) {
  const box = $("thread-note")
  if (!box) return
  box.textContent = text
  box.hidden = !text
}

// --- Dictate --------------------------------------------------------------------------------------

let recorder = null // the recording in progress
let busy = false // a recording being turned into words

function drawMic() {
  const b = $("dictate")
  if (!b) return
  b.hidden = !canDictate()
  const on = !!recorder
  b.classList.toggle("is-recording", on)
  b.classList.toggle("is-busy", busy)
  b.disabled = busy
  b.setAttribute("aria-pressed", String(on))
  const label = busy ? t("Writing down…") : on ? t("Stop dictating") : t("Dictate a message")
  b.setAttribute("aria-label", label)
  b.title = label
}

async function start() {
  say("")
  let stream
  try {
    await invoke("voice_arm")
    stream = await navigator.mediaDevices.getUserMedia({ audio: true })
  } catch (err) {
    say(t("The microphone could not be opened: %(why)s", { why: err?.message || why(err) }))
    return
  }
  const type = ["audio/webm;codecs=opus", "audio/ogg;codecs=opus", "audio/webm"].find((x) => window.MediaRecorder?.isTypeSupported?.(x))
  let rec
  try {
    rec = new MediaRecorder(stream, type ? { mimeType: type } : undefined)
  } catch (err) {
    stream.getTracks().forEach((x) => x.stop())
    say(t("The microphone could not be opened: %(why)s", { why: err?.message || why(err) }))
    return
  }
  const chunks = []
  rec.addEventListener("dataavailable", (e) => e.data.size && chunks.push(e.data))
  rec.addEventListener("stop", async () => {
    stream.getTracks().forEach((x) => x.stop())
    recorder = null
    const blob = new Blob(chunks, { type: rec.mimeType || type || "audio/webm" })
    if (!blob.size) return drawMic()
    busy = true
    drawMic()
    try {
      const text = await invoke("transcribe", new Uint8Array(await blob.arrayBuffer()), { headers: { "x-type": blob.type } })
      intoComposer(text)
    } catch (err) {
      say(why(err))
    }
    busy = false
    drawMic()
  })
  recorder = rec
  rec.start()
  drawMic()
}

/** The words heard, where the cursor is in the message box, a space either side where needed. */
function intoComposer(text) {
  text = String(text || "").trim()
  if (!text) return
  const box = $("composer-text")
  const at = box.selectionStart ?? box.value.length
  const end = box.selectionEnd ?? at
  const before = box.value.slice(0, at)
  const after = box.value.slice(end)
  const words = `${before && !/\s$/.test(before) ? " " : ""}${text}${after && !/^\s/.test(after) ? " " : ""}`
  box.value = before + words + after
  box.selectionStart = box.selectionEnd = before.length + words.length
  box.dispatchEvent(new Event("input"))
  box.focus()
}

/** Stop a recording (a chat closed, the window hidden): what was said is still written down. */
export function stopDictating() {
  if (recorder?.state === "recording") recorder.stop()
}

// --- Read aloud -----------------------------------------------------------------------------------

let playing = null // { id, audio, button }

/** A reply's words without the markdown around them, and without its code. */
export function plain(markdown) {
  return String(markdown || "")
    .replace(/```[\s\S]*?```/g, " ")
    .replace(/!?\[([^\]]*)]\([^)]*\)/g, "$1")
    .replace(/[*_`#>|~]/g, "")
    .replace(/\s+/g, " ")
    .trim()
}

function stopReading() {
  if (!playing) return
  playing.audio?.pause()
  playing.button.setAttribute("aria-pressed", "false")
  playing.button.classList.remove("is-busy")
  playing = null
}

/** The Read aloud button of a reply (`id`), reading `text()` when pressed; null where it cannot. */
export function readAloudButton(id, text) {
  if (!canSpeak()) return null
  const label = t("Read this aloud")
  const b = h("button", { type: "button", class: "msg__act msg__speak", "aria-label": label, title: label, "aria-pressed": "false" }, icon(ICONS.speaker))
  b.addEventListener("click", async () => {
    if (playing?.id === id) return stopReading()
    stopReading()
    const words = plain(text())
    if (!words) return
    const mine = { id, audio: null, button: b }
    playing = mine
    b.setAttribute("aria-pressed", "true")
    b.classList.add("is-busy")
    try {
      const url = await invoke("speak", { text: words })
      if (playing !== mine) return
      // Only audio the app's own server sent, as Rust read it: a data: URL of an audio type.
      if (!/^data:audio\/[a-z0-9.+-]+;base64,/i.test(url)) throw new Error(t("The server's answer was not audio."))
      mine.audio = new Audio(url)
      b.classList.remove("is-busy")
      mine.audio.addEventListener("ended", () => playing === mine && stopReading())
      await mine.audio.play()
    } catch (err) {
      if (playing === mine) stopReading()
      say(why(err?.message ? { detail: err.message } : err))
    }
  })
  return b
}

/** What the account may use, from `GET /me`: the composer's Dictate button is drawn by it. */
export function voiceFor(account) {
  me = account
  drawMic()
}

export function initVoice() {
  const b = $("dictate")
  b.replaceChildren(icon(ICONS.mic))
  b.addEventListener("click", () => {
    if (busy) return
    if (recorder) recorder.stop()
    else start()
  })
  drawMic()
}