diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ceecaf..80db1ab 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,23 @@ All notable changes to LLeMbas CLI. The format follows ## [Unreleased] +## [1.5.1] — 2026-10-23 + +A screenshot no longer makes the session compact. + +### Fixed + +- **An image counted as half a million tokens.** The context estimate measured an image's base64 + as text, so a 1–3 MB screenshot read as "context at 150–280%" the moment `view_image` (or an + `@` attachment) added it, and the session was compacted -- after every image, since the summary + drops pictures and the model looked again. The provider's own count, a few thousand tokens, was + never in question. An image now counts by its dimensions (28-pixel patches, at most 16k tokens; + a flat 1600 when the size cannot be read), everywhere the CLI estimates: the context meter, + pruning, `/context`, and the estimate a provider that reports no usage falls back to. +- **Screenshots are sent at 1600 px.** Any image over 1600 px or 500 KB is shrunk to 1600 px on + its longer edge when ImageMagick is there (only files over 3.5 MB were before): no more detail + than the model is billed for, and a lighter request. + ## [1.5.0] — 2026-10-20 Harness spec 2.0.0: background work is one thing — a job — for commands and subagents alike, and diff --git a/package.json b/package.json index 1be9739..a549495 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "lembas-cli", - "version": "1.5.0", + "version": "1.5.1", "description": "LLeMbas CLI: a terminal coding agent and project manager — OpenTUI, any LLM API, memory, skills, MCP, git-native.", "license": "MIT", "private": true, diff --git a/src/project/image.ts b/src/project/image.ts index 15cee35..d5e87b2 100644 --- a/src/project/image.ts +++ b/src/project/image.ts @@ -1,17 +1,21 @@ // Images for vision models: read, size-checked, downscaled when ImageMagick is there. import { readFileSync, statSync } from "node:fs" import { extname } from "node:path" +import { imageSize } from "../provider/tokens.ts" import type { ImagePart } from "../provider/types.ts" export const IMAGE_TYPES: Record = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp" } const MAX_BYTES = 3_500_000 +const SHRINK_ABOVE_BYTES = 500_000 +const MAX_EDGE = 1600 const magick = Bun.which("magick") ?? Bun.which("convert") export function isImage(path: string): boolean { return extname(path).toLowerCase() in IMAGE_TYPES } -/** The image as a part, or a reason it cannot be one. Large ones are shrunk to 1600 px if possible. */ +/** The image as a part, or a reason it cannot be one. Anything over 1600 px or 500 KB is shrunk to + * 1600 px when ImageMagick is there. */ export function loadImage(path: string): ImagePart | string { const mime = IMAGE_TYPES[extname(path).toLowerCase()] if (!mime) return `${path} is not a supported image (png, jpg, gif, webp)` @@ -21,7 +25,13 @@ export function loadImage(path: string): ImagePart | string { } catch { return `${path} does not exist` } - if (size <= MAX_BYTES) return { type: "image", mime, data: readFileSync(path).toString("base64") } + const bytes = readFileSync(path) + const dims = imageSize(bytes) + // A screenshot of a large monitor is sent at 1600 px on its longer edge, as the LLeMbas server + // scales uploads: a model sees no more detail in 2560 px than it is billed tokens for, and a + // half-megabyte PNG is a slow request for nothing. Without ImageMagick it goes as it is. + const big = size > SHRINK_ABOVE_BYTES || (dims !== undefined && Math.max(dims.width, dims.height) > MAX_EDGE) + if (size <= MAX_BYTES && (!big || !magick)) return { type: "image", mime, data: bytes.toString("base64") } if (!magick) return `${path} is ${(size / 1e6).toFixed(1)} MB — too large to send, and ImageMagick is not installed to shrink it` // The coder named from the extension: ImageMagick otherwise decides by content, and a file named // .png that is really SVG or MVG can make it read other files or fetch URLs. diff --git a/src/provider/anthropic.ts b/src/provider/anthropic.ts index b030ac1..ae5677f 100644 --- a/src/provider/anthropic.ts +++ b/src/provider/anthropic.ts @@ -6,7 +6,7 @@ import { resolveKey } from "../config/load.ts" import { Parts, record, recordFailure, retrying, statusOf, SwapBanner } from "./common.ts" import { authHeaders, joinUrl, request, tlsFor } from "./http.ts" import { sseJson } from "./sse.ts" -import { estimateTokens } from "./tokens.ts" +import { estimateTokens, estimateValue } from "./tokens.ts" import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent, type ToolCallPart } from "./types.ts" /** Thinking budgets by effort, when the model's config gives no effort_map. */ @@ -216,7 +216,7 @@ export class AnthropicClient implements Client { const reason = finish === "max_tokens" ? "length" : finish === "tool_use" || message.parts.some((p) => p.type === "tool_call") ? "tool_calls" : "stop" yield sawUsage ? { type: "usage", usage: { input, output, cached } } - : { type: "usage", usage: { input: estimateTokens(JSON.stringify(body.messages)), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } } + : { type: "usage", usage: { input: estimateValue(body.messages), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } } yield { type: "finish", reason, message } } diff --git a/src/provider/gemini.ts b/src/provider/gemini.ts index e8e5e98..273fc15 100644 --- a/src/provider/gemini.ts +++ b/src/provider/gemini.ts @@ -7,7 +7,7 @@ import { DEFAULT_BUDGETS } from "./anthropic.ts" import { Parts, record, recordFailure, retrying, statusOf } from "./common.ts" import { authHeaders, joinUrl, request, tlsFor } from "./http.ts" import { sseJson } from "./sse.ts" -import { estimateTokens } from "./tokens.ts" +import { estimateTokens, estimateValue } from "./tokens.ts" import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent, type ToolCallPart } from "./types.ts" type Content = { role: "user" | "model"; parts: Record[] } @@ -301,7 +301,7 @@ export class GeminiClient implements Client { } const message = out.message() if (slots.size && finish !== "length") finish = "tool_calls" - yield usage ?? { type: "usage", usage: { input: estimateTokens(JSON.stringify(body.contents)), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } } + yield usage ?? { type: "usage", usage: { input: estimateValue(body.contents), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } } yield { type: "finish", reason: finish, message } } diff --git a/src/provider/ollama.ts b/src/provider/ollama.ts index 8dd6215..c80b4ad 100644 --- a/src/provider/ollama.ts +++ b/src/provider/ollama.ts @@ -7,7 +7,7 @@ import { resolveKey } from "../config/load.ts" import { Parts, record, recordFailure, retrying, statusOf } from "./common.ts" import { authHeaders, joinUrl, request, tlsFor } from "./http.ts" -import { estimateTokens } from "./tokens.ts" +import { estimateTokens, estimateValue } from "./tokens.ts" import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent } from "./types.ts" export function toOllamaMessages(system: string, messages: Message[], vision: boolean): Record[] { @@ -160,7 +160,7 @@ export class OllamaClient implements Client { } const message = out.message() if (calls && finish !== "length") finish = "tool_calls" - yield usage ?? { type: "usage", usage: { input: estimateTokens(JSON.stringify(body.messages)), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } } + yield usage ?? { type: "usage", usage: { input: estimateValue(body.messages), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true } } yield { type: "finish", reason: finish, message } } diff --git a/src/provider/openai-chat.ts b/src/provider/openai-chat.ts index 2354ed1..c51dfc5 100644 --- a/src/provider/openai-chat.ts +++ b/src/provider/openai-chat.ts @@ -11,7 +11,7 @@ import { learned, learnEfforts, learnNoProgress, learnNoStreamOptions } from "./ import { historyArgs, Parts, record, recordFailure, retrying, statusOf, SwapBanner } from "./common.ts" import { sseJson } from "./sse.ts" import { ThinkSplitter } from "./think.ts" -import { estimateTokens } from "./tokens.ts" +import { estimateTokens, estimateValue } from "./tokens.ts" import { ProviderError, type ChatRequest, @@ -274,7 +274,7 @@ export class OpenAIChatClient implements Client { const parts = message.parts if (!usage) { const out = parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.name + p.args : p.text), 0) - usage = { type: "usage", usage: { input: estimateTokens(JSON.stringify(body.messages)), output: out, estimated: true } } + usage = { type: "usage", usage: { input: estimateValue(body.messages), output: out, estimated: true } } } yield usage yield { type: "finish", reason: finish, message } diff --git a/src/provider/responses.ts b/src/provider/responses.ts index 43ead55..0847e7b 100644 --- a/src/provider/responses.ts +++ b/src/provider/responses.ts @@ -6,7 +6,7 @@ import { historyArgs, Parts, record, recordFailure, retrying, statusOf, SwapBann import { authHeaders, joinUrl, request, tlsFor } from "./http.ts" import { contextFrom } from "./openai-chat.ts" import { sseJson } from "./sse.ts" -import { estimateTokens } from "./tokens.ts" +import { estimateTokens, estimateValue } from "./tokens.ts" import { ProviderError, type ChatRequest, type Client, type DiscoveredModel, type Message, type ResolvedModel, type StreamEvent } from "./types.ts" type Item = Record @@ -161,7 +161,7 @@ export class ResponsesClient implements Client { if (finish !== "length" && message.parts.some((p) => p.type === "tool_call")) finish = "tool_calls" yield usage ?? { type: "usage", - usage: { input: estimateTokens(JSON.stringify(body.input)), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true }, + usage: { input: estimateValue(body.input), output: message.parts.reduce((n, p) => n + estimateTokens(p.type === "tool_call" ? p.args : p.text), 0), estimated: true }, } yield { type: "finish", reason: finish, message } } diff --git a/src/provider/tokens.ts b/src/provider/tokens.ts index 72d66fb..0a8a9be 100644 --- a/src/provider/tokens.ts +++ b/src/provider/tokens.ts @@ -4,3 +4,87 @@ export const CHARS_PER_TOKEN = 4 export function estimateTokens(text: string): number { return Math.ceil(text.length / CHARS_PER_TOKEN) } + +/** An image whose size cannot be read counts as this many tokens (a 1600 px screenshot, roughly). */ +export const IMAGE_FALLBACK_TOKENS = 1600 +/** Pixels per token edge: Qwen-VL's 28 px patches; OpenAI's and Anthropic's tiles land near it. */ +const PATCH = 28 +/** No image costs more than this (vLLM's Qwen default max_pixels / 784). */ +const IMAGE_MAX_TOKENS = 16_384 +/** A string this long made only of base64 characters is an image's bytes, not text. */ +const BASE64_MIN = 2_000 +const BASE64 = /^[A-Za-z0-9+/=_-]+$/ + +/** The width and height in an image's first bytes (PNG, JPEG, GIF, WebP), or undefined. */ +export function imageSize(bytes: Uint8Array): { width: number; height: number } | undefined { + const at = (i: number) => bytes[i] ?? 0 + const be16 = (i: number) => (at(i) << 8) | at(i + 1) + const le16 = (i: number) => at(i) | (at(i + 1) << 8) + const be32 = (i: number) => ((at(i) << 24) | (at(i + 1) << 16) | (at(i + 2) << 8) | at(i + 3)) >>> 0 + if (at(0) === 0x89 && at(1) === 0x50 && at(2) === 0x4e && at(3) === 0x47) return { width: be32(16), height: be32(20) } + if (at(0) === 0x47 && at(1) === 0x49 && at(2) === 0x46) return { width: le16(6), height: le16(8) } + if (at(0) === 0x52 && at(1) === 0x49 && at(2) === 0x46 && at(3) === 0x46 && at(8) === 0x57 && at(9) === 0x45) { + const chunk = String.fromCharCode(at(12), at(13), at(14), at(15)) + if (chunk === "VP8X") return { width: 1 + (at(24) | (at(25) << 8) | (at(26) << 16)), height: 1 + (at(27) | (at(28) << 8) | (at(29) << 16)) } + if (chunk === "VP8 ") return { width: le16(26) & 0x3fff, height: le16(28) & 0x3fff } + if (chunk === "VP8L") { + const b = (at(21) | (at(22) << 8) | (at(23) << 16) | (at(24) << 24)) >>> 0 + return { width: (b & 0x3fff) + 1, height: ((b >> 14) & 0x3fff) + 1 } + } + return undefined + } + if (at(0) === 0xff && at(1) === 0xd8) { + // Walk the segments to a start-of-frame marker (C0–CF, except C4, C8, CC). + let i = 2 + while (i + 9 < bytes.length) { + if (at(i) !== 0xff) return undefined + const marker = at(i + 1) + if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) { + return { width: be16(i + 7), height: be16(i + 5) } + } + i += 2 + be16(i + 2) + } + } + return undefined +} + +/** + * What an image costs a model, from its dimensions -- never from its encoding. Counting the + * base64 as text made a 1.5 MB screenshot "512k tokens": the CLI then compacted after every + * view_image, at "context 215%", while the provider charged a few thousand. Only the head is + * decoded (enough for the header, and a JPEG's frame marker in all but odd files). + */ +export function imageTokens(base64: string): number { + const head = base64.replace(/^data:[^,]*,/, "").slice(0, 87_384) // 64 KB of bytes + let size: { width: number; height: number } | undefined + try { + size = imageSize(Buffer.from(head.slice(0, head.length - (head.length % 4)), "base64")) + } catch { + size = undefined + } + if (!size || !size.width || !size.height) return IMAGE_FALLBACK_TOKENS + return Math.min(IMAGE_MAX_TOKENS, Math.ceil(size.width / PATCH) * Math.ceil(size.height / PATCH)) +} + +function isImageData(text: string): boolean { + if (text.startsWith("data:image/")) return true + return text.length >= BASE64_MIN && BASE64.test(text.slice(0, 4096)) +} + +/** + * Tokens for any value about to be sent or kept in history -- a message, a request body in any + * dialect: its JSON measured as text, except that an image is counted by `imageTokens`. Images + * are found by shape, so the one rule covers every wire format (an `image_url` data URL, an + * Anthropic or Gemini base64 `data`, Ollama's `images` list, the history's own `ImagePart`). + */ +export function estimateValue(value: unknown): number { + let images = 0 + const text = JSON.stringify(value, (_key, v) => { + if (typeof v === "string" && isImageData(v)) { + images += imageTokens(v) + return "" + } + return v + }) + return estimateTokens(text ?? "") + images +} diff --git a/src/session/context.ts b/src/session/context.ts index 51975ff..af5597e 100644 --- a/src/session/context.ts +++ b/src/session/context.ts @@ -2,7 +2,7 @@ // up to what the server last reported when it reported anything. import type { Engine } from "./engine.ts" import { contextWindow } from "../provider/discover.ts" -import { estimateTokens } from "../provider/tokens.ts" +import { estimateTokens, imageTokens } from "../provider/tokens.ts" import { toSpec, type Tool } from "../tool/tool.ts" export interface ContextRow { @@ -29,7 +29,7 @@ export function contextBreakdown(engine: Engine, system: string, instructions: { else if (p.type === "tool_call") by.calls += estimateTokens(p.name + p.args) else { imageCount++ - by.images += 800 // providers bill an image at roughly this for a typical screenshot + by.images += imageTokens(p.data) // by its dimensions, as contextUsed counts it } } } diff --git a/src/session/engine.ts b/src/session/engine.ts index 5c3f411..60cdcff 100644 --- a/src/session/engine.ts +++ b/src/session/engine.ts @@ -16,7 +16,7 @@ import { fill, promptText } from "../prompt/assemble.ts" import { resolveCall } from "../tool/names.ts" import { contextWindow } from "../provider/discover.ts" import { summaryTurns, type Store } from "./store.ts" -import { estimateTokens } from "../provider/tokens.ts" +import { estimateTokens, estimateValue } from "../provider/tokens.ts" import { duration } from "../duration.ts" export interface EngineOptions { @@ -244,7 +244,7 @@ export class Engine { * last reply's output but will not be in the next request. */ contextUsed(): number | undefined { if (!this.usage) return undefined - return this.usage.input + this.usage.output - this.droppedThinking() + estimateTokens(JSON.stringify(this.messages.slice(this.usageAt))) + return this.usage.input + this.usage.output - this.droppedThinking() + estimateValue(this.messages.slice(this.usageAt)) } private droppedThinking(): number { @@ -262,7 +262,7 @@ export class Engine { let saved = 0 for (let i = this.messages.length - 1; i >= 0; i--) { const m = this.messages[i]! - const size = estimateTokens(JSON.stringify(m)) + const size = estimateValue(m) if (recent < protect) { recent += size continue diff --git a/tests/image-tokens.test.ts b/tests/image-tokens.test.ts new file mode 100644 index 0000000..0ceab45 --- /dev/null +++ b/tests/image-tokens.test.ts @@ -0,0 +1,67 @@ +import { afterEach, describe, expect, test } from "bun:test" +import { mkdirSync, mkdtempSync, writeFileSync } from "node:fs" +import { tmpdir } from "node:os" +import { join } from "node:path" +import { createApp } from "../src/app.ts" +import type { AskReply, Event } from "../src/bus/index.ts" +import { paths } from "../src/config/paths.ts" +import { estimateValue, IMAGE_FALLBACK_TOKENS, imageSize, imageTokens } from "../src/provider/tokens.ts" +import { delta, fakeProvider, toolCall, usage, type Fake } from "./fake-provider.ts" + +/** A PNG-shaped file of `bytes` bytes that says it is `w`×`h` (header only; the rest is filler). */ +function png(w: number, h: number, bytes: number): Buffer { + const out = Buffer.alloc(bytes, 0x41) + Buffer.from([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a, 0, 0, 0, 13, 0x49, 0x48, 0x44, 0x52]).copy(out, 0) + out.writeUInt32BE(w, 16) + out.writeUInt32BE(h, 20) + return out +} + +let fake: Fake | undefined +afterEach(() => fake?.stop()) + +describe("an image counts by its dimensions, never its encoding", () => { + test("sizes are read from PNG, GIF and JPEG headers", () => { + expect(imageSize(png(2560, 1440, 64))).toEqual({ width: 2560, height: 1440 }) + const gif = Buffer.from([0x47, 0x49, 0x46, 0x38, 0x39, 0x61, 0x40, 0x01, 0xf0, 0x00]) + expect(imageSize(gif)).toEqual({ width: 320, height: 240 }) + // SOI, an APP0 segment of 16 bytes, then SOF0 with height 600 and width 800. + const jpeg = Buffer.from([0xff, 0xd8, 0xff, 0xe0, 0x00, 0x10, ...Array(14).fill(0), 0xff, 0xc0, 0x00, 0x11, 0x08, 0x02, 0x58, 0x03, 0x20, 0x03]) + expect(imageSize(jpeg)).toEqual({ width: 800, height: 600 }) + }) + + test("a 1.5 MB 2560×1440 screenshot is thousands of tokens, not half a million", () => { + const data = png(2560, 1440, 1_500_000).toString("base64") + expect(imageTokens(data)).toBe(Math.ceil(2560 / 28) * Math.ceil(1440 / 28)) + const message = { role: "user", parts: [{ type: "text", text: "look" }, { type: "image", mime: "image/png", data }] } + expect(estimateValue(message)).toBeLessThan(5_000) + // The same image in each dialect's wire shape. + expect(estimateValue([{ content: [{ type: "image_url", image_url: { url: `data:image/png;base64,${data}` } }] }])).toBeLessThan(5_000) + expect(estimateValue([{ content: [{ type: "image", source: { type: "base64", media_type: "image/png", data } }] }])).toBeLessThan(5_000) + expect(estimateValue([{ role: "user", content: "look", images: [data] }])).toBeLessThan(5_000) + }) + + test("an image whose size cannot be read has a flat cost", () => { + expect(imageTokens(Buffer.alloc(300_000, 7).toString("base64"))).toBe(IMAGE_FALLBACK_TOKENS) + }) + + test("view_image of a 1.5 MB screenshot at 20% of the window does not compact", async () => { + fake = fakeProvider([ + { chunks: [toolCall(0, "c1", "view_image", '{"path":"shot.png"}'), usage(52_000, 50)] }, + { chunks: [delta({ content: "a screenshot" }), usage(56_000, 5)] }, + ]) + mkdirSync(paths.config, { recursive: true }) + writeFileSync(join(paths.config, "connections.yaml"), `connections:\n f:\n dialect: openai-chat\n base_url: ${fake.url}\n models:\n m: { vision: true, context: 262144 }\n`, { mode: 0o600 }) + writeFileSync(join(paths.config, "config.yaml"), "model: f/m\n") + const root = mkdtempSync(join(tmpdir(), "ph-img-")) + writeFileSync(join(root, "shot.png"), png(2560, 1440, 1_500_000)) + const app = createApp({ cwd: root, mode: "edit", store: false, asker: { ask: async (): Promise => ({ kind: "once" }) } }) + const notices: string[] = [] + app.bus.on((e: Event) => { + if (e.type === "notice") notices.push(e.message) + }) + await app.engine.prompt("look at shot.png") + expect(notices.filter((n) => n.includes("compacting") || n.includes("pruned"))).toEqual([]) + expect(fake.requests.length).toBe(2) + }) +})