Harness spec 3.2.0 in the CLI: effort levels, reasoning round-trip, no absolute paths
ci / check (push) Waiting to run

Levels (models/schema.json levels): provider/effort.ts levelsOf/levelFields/
readLevel — `off` only with thinking_switch and sends enable_thinking false
and no effort, `default` for a model that takes no efforts and sends
nothing, a level not offered is read as the start level. The engine's
effort is a level; /effort shows nothing for one level, a Thinking switch
for [off, X], a list otherwise; the status bar follows. /v1/models'
thinking_switch, efforts [] and capacity.parallel are read
(lembas/webui.ts); session/capacity.ts counts parallel slots.

Reasoning (context.md §8): an assistant message of the current turn
carries reasoning_content (provider/common.ts turnReasoning,
openai-chat.ts); step-boundary deliveries, nudges and the wrap-up are
marked turn: false and do not start a turn.

Paths (context.md §11): the <env> block names the project and, started
below the root, that directory relative to it (assemble.ts envLines);
instruction files and @ attachments are named from the root or as ~/…;
file tools and bash resolve relative paths against the project root.

The team rules (team/team.ts) and conformance runners for levels,
reasoning, paths and team; the team runtime follows.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
HomerandClaude Opus 5.5 committed 2026-11-09 08:36:55 +00:00
1 parent 5c23087cef
commit fbefe7f844
36 files changed
+1168 -153

No files matched your search

+102
View File
@@ -0,0 +1,102 @@
// Harness spec 3.2.0 through the whole app: effort levels as a request carries them, the model's
// reasoning sent back during its turn and not after it, and no absolute path in what the model is
// told — relative paths and the shell at the project root, wherever the session was started.
import { afterEach, describe, expect, test } from "bun:test"
import { mkdirSync, mkdtempSync, realpathSync, writeFileSync } from "node:fs"
import { tmpdir } from "node:os"
import { join } from "node:path"
import { createApp } from "../src/app.ts"
import type { AskReply } from "../src/bus/index.ts"
import { paths } from "../src/config/paths.ts"
import { delta, fakeProvider, toolCall, usage, type Fake } from "./fake-provider.ts"
let fake: Fake | undefined
afterEach(() => fake?.stop())
function setup(script: Parameters<typeof fakeProvider>[0], spec = "{ context: 32768 }") {
fake = fakeProvider(script)
mkdirSync(paths.config, { recursive: true })
writeFileSync(join(paths.config, "connections.yaml"), `connections:\n fake:\n type: webui\n url: ${fake.url}\n models:\n fake/m: ${spec}\n`, { mode: 0o600 })
writeFileSync(join(paths.config, "config.yaml"), "model: fake/m\n")
const dir = realpathSync(mkdtempSync(join(tmpdir(), "lembas-320-")))
Bun.spawnSync(["git", "init", "-q", dir])
mkdirSync(join(dir, "packages", "web"), { recursive: true })
writeFileSync(join(dir, "top.txt"), "at the root\n")
writeFileSync(join(dir, "packages", "web", "index.ts"), "export {}\n")
return dir
}
const yes = { ask: async (): Promise<AskReply> => ({ kind: "once" }) }
describe("reasoning goes back during its turn (context.md §8)", () => {
test("a step's reasoning is sent back within the turn, a step-boundary note does not end the turn, and the next turn sends none", async () => {
const dir = setup([
{ chunks: [delta({ reasoning_content: "List first." }), toolCall(0, "l1", "list", "{}"), usage(100, 10)] },
{ chunks: [delta({ reasoning_content: "Now answer." }), delta({ content: "Two files." }, "stop"), usage(150, 10)] },
{ chunks: [delta({ content: "Still two." }, "stop"), usage(200, 5)] },
])
const app = createApp({ cwd: dir, mode: "edit", asker: yes, store: false })
// A job's note delivered at the first step boundary: it does not start a turn.
app.engine.note("[A background job you started has finished:]\njob1 (true) exited with 0 after 0s.")
await app.engine.prompt("what is here?")
const second = fake!.requests[1].messages.filter((m: any) => m.role === "assistant")
expect(second).toEqual([expect.objectContaining({ reasoning_content: "List first." })])
await app.engine.prompt("and now?")
// A new turn: the earlier replies go without their reasoning.
const third = fake!.requests[2].messages.filter((m: any) => m.role === "assistant")
expect(third.length).toBe(2)
for (const m of third) expect(m.reasoning_content).toBeUndefined()
})
test("reasoning read from inline <think> tags goes back as reasoning_content, never as text", async () => {
const dir = setup([
{ chunks: [delta({ content: "<think>Look at the list.</think>" }), toolCall(0, "l1", "list", "{}"), usage(100, 10)] },
{ chunks: [delta({ content: "Done." }, "stop"), usage(150, 10)] },
])
const app = createApp({ cwd: dir, mode: "edit", asker: yes, store: false })
await app.engine.prompt("look")
const reply = fake!.requests[1].messages.find((m: any) => m.role === "assistant")
expect(reply.reasoning_content).toBe("Look at the list.")
expect(reply.content ?? "").not.toContain("think")
})
})
describe("effort levels in a request (models/schema.json levels)", () => {
test("off on a model with the switch: enable_thinking false and no effort; the start level otherwise", async () => {
const dir = setup([{ chunks: [delta({ content: "a" }, "stop")] }, { chunks: [delta({ content: "b" }, "stop")] }], "{ thinking_switch: true, efforts: [medium] }")
const app = createApp({ cwd: dir, mode: "edit", asker: yes, store: false, effort: "off" })
expect(app.engine.effort).toBe("off")
await app.engine.prompt("hi")
expect(fake!.requests[0].chat_template_kwargs).toEqual({ enable_thinking: false })
expect(fake!.requests[0].reasoning_effort).toBeUndefined()
app.settings.set("effort", "medium")
await app.engine.prompt("again")
expect(fake!.requests[1].reasoning_effort).toBe("medium")
expect(fake!.requests[1].chat_template_kwargs).toEqual({ reasoning_effort: "medium" })
expect(() => app.settings.set("effort", "high")).toThrow("offers the levels off, medium")
})
test("off asked of a model without the switch starts at its start level", async () => {
const dir = setup([{ chunks: [delta({ content: "a" }, "stop")] }], "{ efforts: [low, high] }")
const app = createApp({ cwd: dir, mode: "edit", asker: yes, store: false, effort: "off" })
expect(app.engine.effort).toBe("low")
})
})
describe("no absolute paths (context.md §11)", () => {
test("started below the root: the prompt names the project and the directory relative to it; read and bash work from the root", async () => {
const dir = setup([
{ chunks: [toolCall(0, "r1", "read", '{"path":"top.txt"}'), toolCall(1, "b1", "bash", '{"command":"pwd","description":"where"}'), usage(100, 10)] },
{ chunks: [delta({ content: "ok" }, "stop"), usage(150, 5)] },
])
const app = createApp({ cwd: join(dir, "packages", "web"), mode: "auto", asker: yes, store: false })
await app.engine.prompt("look")
const system = fake!.requests[0].messages[0].content as string
expect(system).toContain(`Project: ${dir.split("/").pop()} (paths are relative to its root)\nStarted in: packages/web (relative to the project root)`)
expect(system).not.toContain(dir)
const results = fake!.requests[1].messages.filter((m: any) => m.role === "tool").map((m: any) => m.content as string)
expect(results[0]).toContain("1: at the root")
expect(results[1]).toContain(dir)
expect(results[1]).not.toContain("packages/web")
})
})