LLeMbas CLI 1.5.1

An image counted as half a million tokens: the context estimate measured its
base64 as text, so a 1-3 MB screenshot read as "context at 150-280%" the
moment view_image added it, and the session compacted after every image.
An image now counts by its dimensions (28 px patches, at most 16k tokens, a
flat 1600 when unreadable) everywhere the CLI estimates -- the meter, pruning,
/context and the no-usage fallback of all five dialects. Images over 1600 px
or 500 KB are shrunk to 1600 px when ImageMagick is there.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

src/project/image.ts

@@ -1,17 +1,21 @@
 // Images for vision models: read, size-checked, downscaled when ImageMagick is there.
 import { readFileSync, statSync } from "node:fs"
 import { extname } from "node:path"
+import { imageSize } from "../provider/tokens.ts"
 import type { ImagePart } from "../provider/types.ts"
 
 export const IMAGE_TYPES: Record<string, string> = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp" }
 const MAX_BYTES = 3_500_000
+const SHRINK_ABOVE_BYTES = 500_000
+const MAX_EDGE = 1600
 const magick = Bun.which("magick") ?? Bun.which("convert")
 
 export function isImage(path: string): boolean {
   return extname(path).toLowerCase() in IMAGE_TYPES
 }
 
-/** The image as a part, or a reason it cannot be one. Large ones are shrunk to 1600 px if possible. */
+/** The image as a part, or a reason it cannot be one. Anything over 1600 px or 500 KB is shrunk to
+ *  1600 px when ImageMagick is there. */
 export function loadImage(path: string): ImagePart | string {
   const mime = IMAGE_TYPES[extname(path).toLowerCase()]
   if (!mime) return `${path} is not a supported image (png, jpg, gif, webp)`
@@ -21,7 +25,13 @@ export function loadImage(path: string): ImagePart | string {
   } catch {
     return `${path} does not exist`
   }
-  if (size <= MAX_BYTES) return { type: "image", mime, data: readFileSync(path).toString("base64") }
+  const bytes = readFileSync(path)
+  const dims = imageSize(bytes)
+  // A screenshot of a large monitor is sent at 1600 px on its longer edge, as the LLeMbas server
+  // scales uploads: a model sees no more detail in 2560 px than it is billed tokens for, and a
+  // half-megabyte PNG is a slow request for nothing. Without ImageMagick it goes as it is.
+  const big = size > SHRINK_ABOVE_BYTES || (dims !== undefined && Math.max(dims.width, dims.height) > MAX_EDGE)
+  if (size <= MAX_BYTES && (!big || !magick)) return { type: "image", mime, data: bytes.toString("base64") }
   if (!magick) return `${path} is ${(size / 1e6).toFixed(1)} MB — too large to send, and ImageMagick is not installed to shrink it`
   // The coder named from the extension: ImageMagick otherwise decides by content, and a file named
   // .png that is really SVG or MVG can make it read other files or fetch URLs.

src/provider/anthropic.ts

@@ -6,7 +6,7 @@ import { resolveKey } from "../config/load.ts"
 import { Parts, record, recordFailure, retrying, statusOf, SwapBanner } from "./common.ts"
 import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
 import { sseJson } from "./sse.ts"
-import { estimateTokens } from "./tokens.ts"
+import { estimateTokens, estimateValue } from "./tokens.ts"
 import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent, type ToolCallPart } from "./types.ts"
 
 /** Thinking budgets by effort, when the model's config gives no effort_map. */
@@ -216,7 +216,7 @@ export class AnthropicClient implements Client {
     const reason = finish === "max_tokens" ? "length" : finish === "tool_use" || message.parts.some((p) => p.type === "tool_call") ? "tool_calls" : "stop"
     yield sawUsage
       ? { type: "usage", usage: { input, output, cached } }
-      : { type: "usage", usage: { input: estimateTokens(JSON.stringify(body.messages)), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
+      : { type: "usage", usage: { input: estimateValue(body.messages), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
     yield { type: "finish", reason, message }
   }
 

src/provider/gemini.ts

@@ -7,7 +7,7 @@ import { DEFAULT_BUDGETS } from "./anthropic.ts"
 import { Parts, record, recordFailure, retrying, statusOf } from "./common.ts"
 import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
 import { sseJson } from "./sse.ts"
-import { estimateTokens } from "./tokens.ts"
+import { estimateTokens, estimateValue } from "./tokens.ts"
 import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent, type ToolCallPart } from "./types.ts"
 
 type Content = { role: "user" | "model"; parts: Record<string, unknown>[] }
@@ -301,7 +301,7 @@ export class GeminiClient implements Client {
     }
     const message = out.message()
     if (slots.size && finish !== "length") finish = "tool_calls"
-    yield usage ?? { type: "usage", usage: { input: estimateTokens(JSON.stringify(body.contents)), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
+    yield usage ?? { type: "usage", usage: { input: estimateValue(body.contents), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
     yield { type: "finish", reason: finish, message }
   }
 

src/provider/ollama.ts

@@ -7,7 +7,7 @@
 import { resolveKey } from "../config/load.ts"
 import { Parts, record, recordFailure, retrying, statusOf } from "./common.ts"
 import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
-import { estimateTokens } from "./tokens.ts"
+import { estimateTokens, estimateValue } from "./tokens.ts"
 import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent } from "./types.ts"
 
 export function toOllamaMessages(system: string, messages: Message[], vision: boolean): Record<string, unknown>[] {
@@ -160,7 +160,7 @@ export class OllamaClient implements Client {
     }
     const message = out.message()
     if (calls && finish !== "length") finish = "tool_calls"
-    yield usage ?? { type: "usage", usage: { input: estimateTokens(JSON.stringify(body.messages)), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
+    yield usage ?? { type: "usage", usage: { input: estimateValue(body.messages), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } }
     yield { type: "finish", reason: finish, message }
   }
 

src/provider/openai-chat.ts

@@ -11,7 +11,7 @@ import { learned, learnEfforts, learnNoProgress, learnNoStreamOptions } from "./
 import { historyArgs, Parts, record, recordFailure, retrying, statusOf, SwapBanner } from "./common.ts"
 import { sseJson } from "./sse.ts"
 import { ThinkSplitter } from "./think.ts"
-import { estimateTokens } from "./tokens.ts"
+import { estimateTokens, estimateValue } from "./tokens.ts"
 import {
   ProviderError,
   type ChatRequest,
@@ -274,7 +274,7 @@ export class OpenAIChatClient implements Client {
     const parts = message.parts
     if (!usage) {
       const out = parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.name + p.args : p.text), 0)
-      usage = { type: "usage", usage: { input: estimateTokens(JSON.stringify(body.messages)), output: out, estimated: true } }
+      usage = { type: "usage", usage: { input: estimateValue(body.messages), output: out, estimated: true } }
     }
     yield usage
     yield { type: "finish", reason: finish, message }

src/provider/responses.ts

@@ -6,7 +6,7 @@ import { historyArgs, Parts, record, recordFailure, retrying, statusOf, SwapBann
 import { authHeaders, joinUrl, request, tlsFor } from "./http.ts"
 import { contextFrom } from "./openai-chat.ts"
 import { sseJson } from "./sse.ts"
-import { estimateTokens } from "./tokens.ts"
+import { estimateTokens, estimateValue } from "./tokens.ts"
 import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent } from "./types.ts"
 
 type Item = Record<string, unknown>
@@ -161,7 +161,7 @@ export class ResponsesClient implements Client {
     if (finish !== "length" && message.parts.some((p) => p.type === "tool_call")) finish = "tool_calls"
     yield usage ?? {
       type: "usage",
-      usage: { input: estimateTokens(JSON.stringify(body.input)), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true },
+      usage: { input: estimateValue(body.input), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true },
     }
     yield { type: "finish", reason: finish, message }
   }

src/provider/tokens.ts

@@ -4,3 +4,87 @@ export const CHARS_PER_TOKEN = 4
 export function estimateTokens(text: string): number {
   return Math.ceil(text.length / CHARS_PER_TOKEN)
 }
+
+/** An image whose size cannot be read counts as this many tokens (a 1600 px screenshot, roughly). */
+export const IMAGE_FALLBACK_TOKENS = 1600
+/** Pixels per token edge: Qwen-VL's 28 px patches; OpenAI's and Anthropic's tiles land near it. */
+const PATCH = 28
+/** No image costs more than this (vLLM's Qwen default max_pixels / 784). */
+const IMAGE_MAX_TOKENS = 16_384
+/** A string this long made only of base64 characters is an image's bytes, not text. */
+const BASE64_MIN = 2_000
+const BASE64 = /^[A-Za-z0-9+/=_-]+$/
+
+/** The width and height in an image's first bytes (PNG, JPEG, GIF, WebP), or undefined. */
+export function imageSize(bytes: Uint8Array): { width: number; height: number } | undefined {
+  const at = (i: number) => bytes[i] ?? 0
+  const be16 = (i: number) => (at(i) << 8) | at(i + 1)
+  const le16 = (i: number) => at(i) | (at(i + 1) << 8)
+  const be32 = (i: number) => ((at(i) << 24) | (at(i + 1) << 16) | (at(i + 2) << 8) | at(i + 3)) >>> 0
+  if (at(0) === 0x89 && at(1) === 0x50 && at(2) === 0x4e && at(3) === 0x47) return { width: be32(16), height: be32(20) }
+  if (at(0) === 0x47 && at(1) === 0x49 && at(2) === 0x46) return { width: le16(6), height: le16(8) }
+  if (at(0) === 0x52 && at(1) === 0x49 && at(2) === 0x46 && at(3) === 0x46 && at(8) === 0x57 && at(9) === 0x45) {
+    const chunk = String.fromCharCode(at(12), at(13), at(14), at(15))
+    if (chunk === "VP8X") return { width: 1 + (at(24) | (at(25) << 8) | (at(26) << 16)), height: 1 + (at(27) | (at(28) << 8) | (at(29) << 16)) }
+    if (chunk === "VP8 ") return { width: le16(26) & 0x3fff, height: le16(28) & 0x3fff }
+    if (chunk === "VP8L") {
+      const b = (at(21) | (at(22) << 8) | (at(23) << 16) | (at(24) << 24)) >>> 0
+      return { width: (b & 0x3fff) + 1, height: ((b >> 14) & 0x3fff) + 1 }
+    }
+    return undefined
+  }
+  if (at(0) === 0xff && at(1) === 0xd8) {
+    // Walk the segments to a start-of-frame marker (C0–CF, except C4, C8, CC).
+    let i = 2
+    while (i + 9 < bytes.length) {
+      if (at(i) !== 0xff) return undefined
+      const marker = at(i + 1)
+      if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
+        return { width: be16(i + 7), height: be16(i + 5) }
+      }
+      i += 2 + be16(i + 2)
+    }
+  }
+  return undefined
+}
+
+/**
+ * What an image costs a model, from its dimensions -- never from its encoding. Counting the
+ * base64 as text made a 1.5 MB screenshot "512k tokens": the CLI then compacted after every
+ * view_image, at "context 215%", while the provider charged a few thousand. Only the head is
+ * decoded (enough for the header, and a JPEG's frame marker in all but odd files).
+ */
+export function imageTokens(base64: string): number {
+  const head = base64.replace(/^data:[^,]*,/, "").slice(0, 87_384) // 64 KB of bytes
+  let size: { width: number; height: number } | undefined
+  try {
+    size = imageSize(Buffer.from(head.slice(0, head.length - (head.length % 4)), "base64"))
+  } catch {
+    size = undefined
+  }
+  if (!size || !size.width || !size.height) return IMAGE_FALLBACK_TOKENS
+  return Math.min(IMAGE_MAX_TOKENS, Math.ceil(size.width / PATCH) * Math.ceil(size.height / PATCH))
+}
+
+function isImageData(text: string): boolean {
+  if (text.startsWith("data:image/")) return true
+  return text.length >= BASE64_MIN && BASE64.test(text.slice(0, 4096))
+}
+
+/**
+ * Tokens for any value about to be sent or kept in history -- a message, a request body in any
+ * dialect: its JSON measured as text, except that an image is counted by `imageTokens`. Images
+ * are found by shape, so the one rule covers every wire format (an `image_url` data URL, an
+ * Anthropic or Gemini base64 `data`, Ollama's `images` list, the history's own `ImagePart`).
+ */
+export function estimateValue(value: unknown): number {
+  let images = 0
+  const text = JSON.stringify(value, (_key, v) => {
+    if (typeof v === "string" && isImageData(v)) {
+      images += imageTokens(v)
+      return ""
+    }
+    return v
+  })
+  return estimateTokens(text ?? "") + images
+}

src/session/context.ts

@@ -2,7 +2,7 @@
 // up to what the server last reported when it reported anything.
 import type { Engine } from "./engine.ts"
 import { contextWindow } from "../provider/discover.ts"
-import { estimateTokens } from "../provider/tokens.ts"
+import { estimateTokens, imageTokens } from "../provider/tokens.ts"
 import { toSpec, type Tool } from "../tool/tool.ts"
 
 export interface ContextRow {
@@ -29,7 +29,7 @@ export function contextBreakdown(engine: Engine, system: string, instructions: {
         else if (p.type === "tool_call") by.calls += estimateTokens(p.name + p.args)
         else {
           imageCount++
-          by.images += 800 // providers bill an image at roughly this for a typical screenshot
+          by.images += imageTokens(p.data) // by its dimensions, as contextUsed counts it
         }
       }
   }

src/session/engine.ts

@@ -16,7 +16,7 @@ import { fill, promptText } from "../prompt/assemble.ts"
 import { resolveCall } from "../tool/names.ts"
 import { contextWindow } from "../provider/discover.ts"
 import { summaryTurns, type Store } from "./store.ts"
-import { estimateTokens } from "../provider/tokens.ts"
+import { estimateTokens, estimateValue } from "../provider/tokens.ts"
 import { duration } from "../duration.ts"
 
 export interface EngineOptions {
@@ -244,7 +244,7 @@ export class Engine {
    *  last reply's output but will not be in the next request. */
   contextUsed(): number | undefined {
     if (!this.usage) return undefined
-    return this.usage.input + this.usage.output - this.droppedThinking() + estimateTokens(JSON.stringify(this.messages.slice(this.usageAt)))
+    return this.usage.input + this.usage.output - this.droppedThinking() + estimateValue(this.messages.slice(this.usageAt))
   }
 
   private droppedThinking(): number {
@@ -262,7 +262,7 @@ export class Engine {
     let saved = 0
     for (let i = this.messages.length - 1; i >= 0; i--) {
       const m = this.messages[i]!
-      const size = estimateTokens(JSON.stringify(m))
+      const size = estimateValue(m)
       if (recent < protect) {
         recent += size
         continue