Voice (dictate, read aloud) and Draw a picture over /v1, with the app's models scope

The server kept voice and images on /v1 (models scope, which the app's token
already asks for); /me's services and scopes decide what is offered. Rust
sends the recording to /v1/audio/transcriptions, reads /v1/audio/speech back
as a data: URL (CSP media-src data:), and for /v1/images/generations takes
only the id of each of the person's own files from the answer and fetches
/v1/files/<id>/content from the app's own server. The microphone is allowed
only within seconds of the page's own Dictate (voice_arm); WebKitGTK's
media stream is switched on through its settings getter.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src-tauri/src/lib.rs

@@ -36,6 +36,7 @@ use serde::Serialize;
 use serde_json::Value;
 use tauri::menu::{Menu, MenuItem, PredefinedMenuItem};
 use tauri::tray::TrayIconBuilder;
+use tauri::webview::{PermissionKind, PermissionResponse};
 use tauri::window::Color;
 use tauri::{
     AppHandle, DragDropEvent, Emitter, LogicalPosition, Manager, RunEvent, Url, WebviewUrl,
@@ -122,6 +123,9 @@ struct App {
     watch: watch::Watch,
     /// The files last dropped on the window: what `attach_dropped` may send, and nothing else.
     dropped: Mutex<Vec<PathBuf>>,
+    /// Until when the page's own Dictate button may have the microphone (`voice_arm`): a request
+    /// for it at any other moment is refused.
+    mic_until: Mutex<Option<std::time::Instant>>,
 }
 
 struct TrayItems {
@@ -153,6 +157,13 @@ impl App {
         Some((server.app, account.token))
     }
 
+    /// The server's own address and the app's token: for the `/v1` routes (voice, pictures).
+    fn door_v1(&self) -> Option<(String, String)> {
+        let server = self.server.lock().unwrap().clone()?;
+        let account = self.account.lock().unwrap().clone()?;
+        Some((server.base, account.token))
+    }
+
     /// The folders the agent may use: what the explorer and the viewer may read.
     fn roots(&self) -> Vec<String> {
         self.remote
@@ -452,6 +463,77 @@ async fn model_image(handle: AppHandle, path: String) -> Result<String, ApiError
         .map_err(|e| ApiError::new(0, e.to_string()))?
 }
 
+// --- Voice and pictures (`/v1`, the app's token holds the `models` scope) -----------------------
+
+/// Run a `/v1` call off the page's thread, as the signed-in account; a 401 signs the app out.
+async fn on_v1<T: Send + 'static>(
+    handle: &AppHandle,
+    f: impl FnOnce(&str, &str) -> Result<T, ApiError> + Send + 'static,
+) -> Result<T, ApiError> {
+    let Some((base, token)) = handle.state::<App>().door_v1() else {
+        return Err(ApiError::new(401, "Not signed in."));
+    };
+    let result = tauri::async_runtime::spawn_blocking(move || f(&base, &token))
+        .await
+        .map_err(|e| ApiError::new(0, e.to_string()))?;
+    if matches!(&result, Err(e) if e.status == 401) {
+        token_refused(handle);
+    }
+    result
+}
+
+/// The page is about to ask for the microphone, because the person pressed Dictate: for the
+/// next few seconds a request for it is allowed (`permission`), and at no other time.
+#[tauri::command]
+fn voice_arm(app: tauri::State<'_, App>) {
+    *app.mic_until.lock().unwrap() =
+        Some(std::time::Instant::now() + std::time::Duration::from_secs(10));
+}
+
+/// A recording made in the composer, in words: its bytes as the raw body of the call, its media
+/// type in `x-type`.
+#[tauri::command]
+async fn transcribe(
+    handle: AppHandle,
+    request: tauri::ipc::Request<'_>,
+) -> Result<String, ApiError> {
+    let tauri::ipc::InvokeBody::Raw(bytes) = request.body() else {
+        return Err(ApiError::new(400, "No recording."));
+    };
+    if bytes.len() as u64 > server::RECORDING_LIMIT {
+        return Err(ApiError::new(413, "That recording is too long."));
+    }
+    let kind = request
+        .headers()
+        .get("x-type")
+        .and_then(|v| v.to_str().ok())
+        .unwrap_or("")
+        .to_string();
+    let bytes = bytes.clone();
+    on_v1(&handle, move |base, token| {
+        server::transcribe(base, token, &kind, &bytes)
+    })
+    .await
+}
+
+/// A reply read aloud, as a `data:` URL of its audio.
+#[tauri::command]
+async fn speak(handle: AppHandle, text: String) -> Result<String, ApiError> {
+    on_v1(&handle, move |base, token| {
+        server::speak(base, token, &text)
+    })
+    .await
+}
+
+/// Pictures drawn by the instance for `prompt`, as `data:` URLs.
+#[tauri::command]
+async fn draw(handle: AppHandle, prompt: String, size: String) -> Result<Vec<String>, ApiError> {
+    on_v1(&handle, move |base, token| {
+        server::draw(base, token, &prompt, &size)
+    })
+    .await
+}
+
 /// How the CLI is started in a terminal here (`lembas`, or the bundled one by its path): the
 /// first word of "continue in a terminal" for an agent chat on this computer.
 #[tauri::command]
@@ -1249,7 +1331,39 @@ fn build_main(handle: &AppHandle) -> tauri::Result<WebviewWindow> {
         .decorations(false)
         .background_color(theme.background())
         .on_navigation(allowed)
+        .on_permission_request(permission)
         .build()
+        .inspect(media_stream)
+}
+
+/// WebKitGTK answers no permission request its app does not, and has no prompt of its own. The
+/// microphone is allowed when the page's own Dictate button has just asked for it (`voice_arm`,
+/// which only the app's page can call: sealed frames have no bridge), once; nothing else is.
+fn permission(view: tauri::Webview, kind: PermissionKind) -> PermissionResponse {
+    if kind != PermissionKind::Microphone {
+        return PermissionResponse::Default;
+    }
+    let app = view.app_handle().state::<App>();
+    let armed = app.mic_until.lock().unwrap().take();
+    if armed.is_some_and(|until| std::time::Instant::now() <= until) {
+        PermissionResponse::Allow
+    } else {
+        PermissionResponse::Deny
+    }
+}
+
+/// `getUserMedia` is off in WebKitGTK unless the app turns it on (it is on in other webviews).
+/// The settings are read with their own getter: the `settings` property is write-only.
+fn media_stream(window: &WebviewWindow) {
+    #[cfg(target_os = "linux")]
+    let _ = window.with_webview(|pv| {
+        use webkit2gtk::{SettingsExt, WebViewExt};
+        if let Some(settings) = pv.inner().settings() {
+            settings.set_enable_media_stream(true);
+        }
+    });
+    #[cfg(not(target_os = "linux"))]
+    let _ = window;
 }
 
 /// The window shows the app's own page, and the sealed frames inside it; nothing else, ever.
@@ -1710,6 +1824,10 @@ pub fn run() {
             sealed_close,
             attach_pick,
             library_upload,
+            voice_arm,
+            transcribe,
+            speak,
+            draw,
             attach_dropped,
             attach_bytes,
             set_theme,
@@ -1770,6 +1888,7 @@ pub fn run() {
                 sealed: sealed::Pages::default(),
                 watch: watch::Watch::default(),
                 dropped: Mutex::new(Vec::new()),
+                mic_until: Mutex::new(None),
             });
             // Without a tray (no AppIndicator library, no StatusNotifier host) the app still
             // starts: the title bar's menu has every item the tray has.

src-tauri/src/server.rs

@@ -652,10 +652,15 @@ fn header_name(name: &str) -> String {
 
 /// The body of a `multipart/form-data` request with one file in the field `field`.
 fn multipart(boundary: &str, field: &str, name: &str, bytes: &[u8]) -> Vec<u8> {
+    multipart_typed(boundary, field, name, "application/octet-stream", bytes)
+}
+
+/// The same, with the file's own media type (a recording says what it is: `audio/webm`).
+fn multipart_typed(boundary: &str, field: &str, name: &str, kind: &str, bytes: &[u8]) -> Vec<u8> {
     let mut body = Vec::with_capacity(bytes.len() + 256);
     body.extend_from_slice(
         format!(
-            "--{boundary}\r\nContent-Disposition: form-data; name=\"{field}\"; filename=\"{}\"\r\nContent-Type: application/octet-stream\r\n\r\n",
+            "--{boundary}\r\nContent-Disposition: form-data; name=\"{field}\"; filename=\"{}\"\r\nContent-Type: {kind}\r\n\r\n",
             header_name(name)
         )
         .as_bytes(),
@@ -707,6 +712,192 @@ pub fn upload_to_library(
     send_file(app, token, &path, "files", name, bytes)
 }
 
+// --- Voice and pictures: the OpenAI-compatible routes of `/v1`, under the `models` scope ------------
+
+/// The most a recording may be here: the server's own limit is 25 MB.
+pub const RECORDING_LIMIT: u64 = 25 * 1024 * 1024;
+/// The most a reply read aloud may be, as audio.
+pub const SPEECH_LIMIT: u64 = 32 * 1024 * 1024;
+/// The most a generated picture may be.
+pub const DRAWN_LIMIT: u64 = 24 * 1024 * 1024;
+/// The media types a recording may say it is; anything else goes as `audio/webm`.
+const RECORDING_TYPES: [&str; 5] = [
+    "audio/webm",
+    "audio/ogg",
+    "audio/wav",
+    "audio/mpeg",
+    "audio/mp4",
+];
+
+/// A recording's media type as it may be sent: one of ours, its parameters (`;codecs=opus`) off.
+pub fn recording_type(kind: &str) -> &'static str {
+    let bare = kind
+        .split(';')
+        .next()
+        .unwrap_or("")
+        .trim()
+        .to_ascii_lowercase();
+    RECORDING_TYPES
+        .iter()
+        .find(|t| **t == bare)
+        .copied()
+        .unwrap_or("audio/webm")
+}
+
+/// A recording, in words: `POST /v1/audio/transcriptions` (the instance's own model, the
+/// account's language). The text the server heard.
+pub fn transcribe(base: &str, token: &str, kind: &str, bytes: &[u8]) -> Result<String, ApiError> {
+    if bytes.is_empty() {
+        return Err(ApiError::new(400, "The recording was empty."));
+    }
+    let kind = recording_type(kind);
+    let ext = kind
+        .rsplit('/')
+        .next()
+        .unwrap_or("webm")
+        .replace("mpeg", "mp3");
+    let nanos = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .map(|d| d.as_nanos())
+        .unwrap_or(0);
+    let boundary = format!("llembas-desktop-{nanos:x}");
+    let res = agent(Duration::from_secs(300))
+        .post(&format!("{base}/v1/audio/transcriptions"))
+        .set("authorization", &format!("Bearer {token}"))
+        .set("accept", "application/json")
+        .set(
+            "content-type",
+            &format!("multipart/form-data; boundary={boundary}"),
+        )
+        .send_bytes(&multipart_typed(
+            &boundary,
+            "file",
+            &format!("speech.{ext}"),
+            kind,
+            bytes,
+        ))?;
+    let v: Value = res
+        .into_json()
+        .map_err(|_| ApiError::new(0, "The server's answer was not JSON.".to_string()))?;
+    Ok(v.get("text")
+        .and_then(Value::as_str)
+        .unwrap_or("")
+        .to_string())
+}
+
+/// `bytes` of a media type of `kinds` (by its prefix), as a `data:` URL; at most `limit`.
+fn as_data_url(res: ureq::Response, prefix: &str, limit: u64) -> Result<String, ApiError> {
+    let kind = res
+        .content_type()
+        .split(';')
+        .next()
+        .unwrap_or("")
+        .trim()
+        .to_ascii_lowercase();
+    let fits = if prefix == "image/" {
+        IMAGE_TYPES.contains(&kind.as_str())
+    } else {
+        kind.starts_with(prefix)
+            && kind
+                .chars()
+                .all(|c| c.is_ascii_alphanumeric() || "/-+.".contains(c))
+    };
+    if !fits {
+        return Err(ApiError::new(
+            415,
+            format!("Not what was asked for: {kind}"),
+        ));
+    }
+    let mut bytes = Vec::new();
+    res.into_reader()
+        .take(limit)
+        .read_to_end(&mut bytes)
+        .map_err(|e| ApiError::new(0, format!("The server's answer could not be read: {e}")))?;
+    if bytes.len() as u64 >= limit {
+        return Err(ApiError::new(413, "The server's answer is too large."));
+    }
+    Ok(format!("data:{kind};base64,{}", base64(&bytes)))
+}
+
+/// A reply read aloud: `POST /v1/audio/speech` in the account's own voice and speed. The audio,
+/// as a `data:` URL the page plays.
+pub fn speak(base: &str, token: &str, text: &str) -> Result<String, ApiError> {
+    let text = text.trim();
+    if text.is_empty() {
+        return Err(ApiError::new(400, "There is nothing to read aloud."));
+    }
+    let res = agent(Duration::from_secs(300))
+        .post(&format!("{base}/v1/audio/speech"))
+        .set("authorization", &format!("Bearer {token}"))
+        .send_json(serde_json::json!({ "input": text, "response_format": "mp3" }))?;
+    as_data_url(res, "audio/", SPEECH_LIMIT)
+}
+
+/// The id of one of the person's own files in a URL `/v1/images/generations` answered with:
+/// `…/v1/files/<id>/content`, and nothing else. The picture is then fetched from this app's own
+/// server by that id -- never from the address the answer named.
+pub fn own_file_id(url: &str) -> Option<String> {
+    let rest = url.split(['?', '#']).next()?;
+    let tail = rest.strip_suffix("/content")?;
+    let (head, id) = tail.rsplit_once('/')?;
+    let ok = head.ends_with("/v1/files")
+        && !id.is_empty()
+        && id.len() <= 128
+        && id
+            .chars()
+            .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_');
+    ok.then(|| id.to_string())
+}
+
+/// Sizes a picture may be asked for: the instance rounds to what its workflow takes.
+const SIZES: [&str; 3] = ["1024x1024", "1344x768", "768x1344"];
+
+/// Pictures drawn by the instance: `POST /v1/images/generations` (its own workflow and default
+/// checkpoint), each then read from the person's own files on this server. `data:` URLs.
+pub fn draw(base: &str, token: &str, prompt: &str, size: &str) -> Result<Vec<String>, ApiError> {
+    let prompt = prompt.trim();
+    if prompt.is_empty() {
+        return Err(ApiError::new(400, "Say what to draw."));
+    }
+    let mut body = serde_json::json!({ "prompt": prompt, "n": 1, "response_format": "url" });
+    if SIZES.contains(&size) {
+        body["size"] = Value::String(size.to_string());
+    }
+    let res = agent(Duration::from_secs(900))
+        .post(&format!("{base}/v1/images/generations"))
+        .set("authorization", &format!("Bearer {token}"))
+        .set("accept", "application/json")
+        .send_json(body)?;
+    let v: Value = res
+        .into_json()
+        .map_err(|_| ApiError::new(0, "The server's answer was not JSON.".to_string()))?;
+    let ids: Vec<String> = v
+        .get("data")
+        .and_then(Value::as_array)
+        .map(|items| {
+            items
+                .iter()
+                .filter_map(|i| i.get("url").and_then(Value::as_str).and_then(own_file_id))
+                .collect()
+        })
+        .unwrap_or_default();
+    if ids.is_empty() {
+        return Err(ApiError::new(
+            0,
+            "The server's answer had no picture of yours in it.",
+        ));
+    }
+    ids.iter()
+        .map(|id| {
+            let res = agent(Duration::from_secs(60))
+                .get(&format!("{base}/v1/files/{id}/content"))
+                .set("authorization", &format!("Bearer {token}"))
+                .call()?;
+            as_data_url(res, "image/", DRAWN_LIMIT)
+        })
+        .collect()
+}
+
 /// One `multipart/form-data` POST of one file to `path` on the app's door; the JSON answer.
 fn send_file(
     app: &str,
@@ -985,6 +1176,42 @@ mod tests {
         assert_eq!(base64(&[0xfb, 0xff]), "+/8=");
     }
 
+    #[test]
+    fn a_drawn_picture_is_read_only_as_one_of_the_persons_own_files() {
+        assert_eq!(
+            own_file_id("https://ai.example/v1/files/abc_12-3/content").as_deref(),
+            Some("abc_12-3")
+        );
+        assert_eq!(
+            own_file_id("http://other/v1/files/x/content?t=1").as_deref(),
+            Some("x")
+        );
+        assert_eq!(own_file_id("https://ai.example/v1/files/../content"), None);
+        assert_eq!(own_file_id("https://ai.example/v1/files//content"), None);
+        assert_eq!(own_file_id("https://ai.example/uploads/x/content"), None);
+        assert_eq!(
+            own_file_id("https://ai.example/v1/files/a%2Fb/content"),
+            None
+        );
+        assert_eq!(own_file_id("javascript:alert(1)"), None);
+    }
+
+    #[test]
+    fn a_recording_says_one_of_the_known_types() {
+        assert_eq!(recording_type("audio/webm;codecs=opus"), "audio/webm");
+        assert_eq!(recording_type("Audio/OGG"), "audio/ogg");
+        assert_eq!(recording_type("text/html"), "audio/webm");
+        let body = String::from_utf8(multipart_typed(
+            "B",
+            "file",
+            "speech.webm",
+            "audio/webm",
+            b"x",
+        ))
+        .unwrap();
+        assert!(body.contains("filename=\"speech.webm\"\r\nContent-Type: audio/webm\r\n"));
+    }
+
     #[test]
     fn a_file_goes_as_one_multipart_field() {
         let body = multipart("XYZ", "file", "a\"b\r\n.txt", b"hello");