A desktop coding agent with free access to frontier models.
TypeScript
fix(pdf): page images for PDFs the agent finds in the workspace (#60)
Skandesh1e328a8f2d5e
diff --git a/apps/server/src/opencode-plugins/omnirush-engine2.ts b/apps/server/src/opencode-plugins/omnirush-engine2.tsindex 94854c53..5c5f1856 100644--- a/apps/server/src/opencode-plugins/omnirush-engine2.ts+++ b/apps/server/src/opencode-plugins/omnirush-engine2.ts@@ -369,7 +369,8 @@ export default {if (messageHooks.length && Array.isArray(event.messages)) {const { v1, restore } = messagesForV1(event.messages);const output = { messages: v1 };- for (const hook of messageHooks) await hook({ sessionID: input.sessionID, model: input.model }, output);+ // `agent` names title and compaction requests, which may run on another model.+ for (const hook of messageHooks) await hook({ sessionID: input.sessionID, model: input.model, agent: input.agent }, output);event.messages = restore(output.messages as Rec[]);}if (paramHooks.length && isRec(event.options)) {diff --git a/apps/server/src/opencode-plugins/omnirush-pdf-attachments.test.ts b/apps/server/src/opencode-plugins/omnirush-pdf-attachments.test.tsindex df318079..a3cac1b0 100644--- a/apps/server/src/opencode-plugins/omnirush-pdf-attachments.test.ts+++ b/apps/server/src/opencode-plugins/omnirush-pdf-attachments.test.ts@@ -8,6 +8,7 @@ import { buildOmniRushRuntimeConfigObject } from "../omnirush-runtime-config.js"import { omnirushPdfAttachmentsPluginPath } from "../omnirush-extensions-plugin-path.js";import { resetDerivedPdfMemory } from "../pdf-attachments/derive.js";import { buildTestPdf, corruptTestPdf, pdfDataUrl } from "../pdf-attachments/pdf-fixture.test-helper.js";+import { withinImageBudget } from "../pdf-attachments/capabilities.js";import { OmniRushPdfAttachments } from "./omnirush-pdf-attachments.js";const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), "../..");@@ -23,6 +24,7 @@ const catalog = {{ id: "openrouter", npm: "@ai-sdk/openai-compatible", models: { vision: { id: "vision", attachment: true, modalities: { input: ["text", "image"], output: ["text"] } } } },{ id: "ollama", npm: "@ai-sdk/openai-compatible", models: { text: { id: "text", attachment: false, modalities: { input: ["text"], output: ["text"] } } } },{ id: "odd", npm: "@ai-sdk/openai-compatible", models: { "pdf-no-vision": { id: "pdf-no-vision", attachment: true, modalities: { input: ["text", "pdf"], output: ["text"] } } } },+ { id: "omnirush", models: { "gpt-6-astra": { id: "gpt-6-astra", providerID: "omnirush", api: { id: "gpt-6-astra", url: "", npm: "@ai-sdk/openai" }, limit: { context: 400000, output: 128000 }, capabilities: { attachment: true, toolcall: true, input: { text: true, audio: false, image: true, video: false, pdf: true }, output: { text: true, audio: false, image: false, video: false, pdf: false } } } } },],default: {},connected: [],@@ -35,6 +37,7 @@ const NATIVE_100_PAGES: Model = { providerID: "openai", modelID: "gpt-native" };const VISION: Model = { providerID: "openrouter", modelID: "vision" };const TEXT: Model = { providerID: "ollama", modelID: "text" };const UNLISTED: Model = { providerID: "custom", modelID: "mystery" };+const ASTRA: Model = { providerID: "omnirush", modelID: "gpt-6-astra" };function isRecord(value: unknown): value is Record<string, unknown> {return typeof value === "object" && value !== null && !Array.isArray(value);@@ -461,6 +464,91 @@ describe("OmniRush.ai PDF attachments plugin", () => {});}, 30_000);+ test("the page tool renders pages of a workspace PDF no chat message attached, for an image-capable session", async () => {+ // Production 2026-09-30: agents open PDFs they find in the project with the+ // page tool; with no PDF in the chat the session's model was never+ // recorded, so image models got the text-only answer and no pages.+ await withWorkspace(async (root) => {+ await mkdir(join(root, "docs"), { recursive: true });+ await writeFile(join(root, "docs", "spec.pdf"), buildTestPdf(["Spec page one", "Spec page two"]));+ const plugin = await OmniRushPdfAttachments({ directory: root, client: { provider: { list: async () => catalog } } });+ const tool = plugin.tool.omnirush_pdf_pages;++ // Session ids no other test uses: the plugin's session memory is module-wide.+ const visionStep = { messages: [userMessage(VISION, [question], "mw1")] };+ visionStep.messages[0].info.sessionID = "ses-ws-vision";+ await plugin["experimental.chat.messages.transform"]({}, visionStep);+ const visionResult = await tool.execute({ pdf_path: "docs/spec.pdf", pages: [1, 2] }, { sessionID: "ses-ws-vision" });+ if (typeof visionResult === "string") throw new Error(`Expected page images, got: ${visionResult}`);+ expect(visionResult.attachments.map((attachment) => attachment.mime)).toEqual(["image/png", "image/png"]);+ expect(visionResult.output).toContain("page_images_attached: pages 1-2, in order");++ const textStep = { messages: [userMessage(TEXT, [question], "mw2")] };+ textStep.messages[0].info.sessionID = "ses-ws-text";+ await plugin["experimental.chat.messages.transform"]({}, textStep);+ const textResult = await tool.execute({ pdf_path: "docs/spec.pdf", pages: [1] }, { sessionID: "ses-ws-text" });+ expect(String(textResult)).toContain("page_images_attached: none (this model cannot view images; text is provided instead)");+ });+ }, 30_000);++ test("on the 2.x engine the step's model comes from the hook input, since the bridge's messages carry no model", async () => {+ // omnirush-engine2.ts messagesForV1 gives each message only {id, role} and+ // passes {sessionID, model} as the hook input.+ await withWorkspace(async (root) => {+ await mkdir(join(root, "docs"), { recursive: true });+ await writeFile(join(root, "docs", "spec.pdf"), buildTestPdf(["Spec page one", "Spec page two"]));+ const plugin = await OmniRushPdfAttachments({ directory: root, client: { provider: { list: async () => catalog } } });+ const bridged = { messages: [{ info: { id: "m0", role: "user" }, content: [{ type: "text", text: "Read docs/spec.pdf" }] }] };+ await plugin["experimental.chat.messages.transform"]({ sessionID: "ses-v2", model: { id: "vision", providerID: "openrouter" }, agent: "build" }, bridged);+ // A title request on a small text-only model must not replace the session's model.+ await plugin["experimental.chat.messages.transform"]({ sessionID: "ses-v2", model: { id: "text", providerID: "ollama" }, agent: "title" }, structuredClone(bridged));+ const result = await plugin.tool.omnirush_pdf_pages.execute({ pdf_path: "docs/spec.pdf", pages: [1, 2] }, { sessionID: "ses-v2" });+ if (typeof result === "string") throw new Error(`Expected page images, got: ${result}`);+ expect(result.attachments.map((attachment) => attachment.mime)).toEqual(["image/png", "image/png"]);+ expect(result.output).toContain("page_images_attached: pages 1-2, in order");+ });+ }, 30_000);++ test("an omnirush.ai model gets a PDF as page images and text, never the PDF file itself", async () => {+ await withWorkspace(async (root) => {+ const pdf = buildTestPdf(["Quarterly revenue report", "Appendix: totals"]);+ const step = { messages: [userMessage(ASTRA, [pdfPart(pdfDataUrl(pdf)), question], "ma1")] };+ step.messages[0].info.sessionID = "ses-astra";+ const [message] = await transform(root, step.messages);+ const parts = partsOf(message);+ expect(parts.some((part) => part.mime === "application/pdf")).toBe(false);+ expect(parts.filter((part) => part.type === "file" && part.mime === "image/png").length).toBe(2);+ expect(noteOf(message)).toContain("page_images_in_this_message: pages 1-2, in order");+ });+ }, 30_000);++ test("a session that keeps stepping is never evicted from the plugin's session memory", async () => {+ await withWorkspace(async (root) => {+ await mkdir(join(root, "docs"), { recursive: true });+ await writeFile(join(root, "docs", "spec.pdf"), buildTestPdf(["Spec page one"]));+ const plugin = await OmniRushPdfAttachments({ directory: root, client: { provider: { list: async () => catalog } } });+ const step = async (sessionID: string) => {+ const output = { messages: [userMessage(VISION, [question], `m-${sessionID}`)] };+ output.messages[0].info.sessionID = sessionID;+ await plugin["experimental.chat.messages.transform"]({}, output);+ };+ await step("ses-busy");+ for (let index = 0; index < 200; index += 1) await step(`ses-other-${index}`);+ await step("ses-busy");+ for (let index = 200; index < 300; index += 1) await step(`ses-other-${index}`);+ const result = await plugin.tool.omnirush_pdf_pages.execute({ pdf_path: "docs/spec.pdf", pages: [1] }, { sessionID: "ses-busy" });+ if (typeof result === "string") throw new Error(`Expected page images, got: ${result}`);+ expect(result.output).toContain("page_images_attached: pages 1, in order");+ });+ }, 30_000);++ test("page images stop at the route's byte budget, in page order", () => {+ const pages = [{ page: 1, bytes: 800 }, { page: 2, bytes: 700 }, { page: 3, bytes: 900 }];+ expect(withinImageBudget(pages, undefined).map((image) => image.page)).toEqual([1, 2, 3]);+ expect(withinImageBudget(pages, 1_600).map((image) => image.page)).toEqual([1, 2]);+ expect(withinImageBudget(pages, 500)).toEqual([]);+ });+test("is registered in runtime config, bundled with its wasm runtime, and packaged by the desktop app", async () => {const runtime = await buildOmniRushRuntimeConfigObject();const plugin = runtime.plugin;diff --git a/apps/server/src/opencode-plugins/omnirush-pdf-attachments.ts b/apps/server/src/opencode-plugins/omnirush-pdf-attachments.tsindex b142e1b9..9cc037f3 100644--- a/apps/server/src/opencode-plugins/omnirush-pdf-attachments.ts+++ b/apps/server/src/opencode-plugins/omnirush-pdf-attachments.ts@@ -2,7 +2,7 @@ import { realpath } from "node:fs/promises";import { basename, isAbsolute, relative, resolve } from "node:path";import { fileURLToPath } from "node:url";import { z } from "zod";-import { createInputSupportResolver, encodedSize, nativePdfPolicy, TEXT_ONLY } from "../pdf-attachments/capabilities.js";+import { createInputSupportResolver, encodedSize, nativePdfPolicy, TEXT_ONLY, withinImageBudget } from "../pdf-attachments/capabilities.js";import type { InputSupportResolver, ModelInputSupport, NativePdfPolicy } from "../pdf-attachments/capabilities.js";import {cachedDerivedPdf,@@ -45,6 +45,19 @@ const MAX_PDF_BYTES = 64 * MIB;const MAX_INLINE_PAGES = EAGER_RENDERED_PAGES;/** Total inline image bytes per PDF; keeps requests under provider payload limits. */const INLINE_IMAGE_BUDGET_BYTES = 12 * MIB;+/** The omnirush.ai models (omnirush-runtime-config's internal provider). */+const OMNIRUSH_PROVIDER_ID = "omnirush";+/**+ * Page-image bytes per PDF (and per page-tool call) for an omnirush.ai model.+ * The omnirush.ai gateway refuses a request body over 8 MiB+ * (`model_request_too_large`) and images ride base64-encoded in every later+ * step too, so 2 MiB (about 2.7 MiB encoded) leaves room for the rest.+ */+const OMNIRUSH_IMAGE_BUDGET_BYTES = 2 * MIB;+/** Sessions whose model the plugin remembers; the least recently used goes first. */+const MAX_REMEMBERED_SESSIONS = 256;+/** Side requests that may run on another (small) model; their model is not the session's. */+const SIDE_AGENTS = new Set(["title", "compaction", "summary"]);/** Extracted text inlined in the note; the full text stays on disk. */const MAX_INLINE_TEXT_CHARS = 60_000;const PAGE_TOOL_NAME = "omnirush_pdf_pages";@@ -364,8 +377,24 @@ const digestByKey = new Map<string, string>();function scopedMemoKey(root: string | null, key: string): string {return `${root ?? ""}\n${key}`;}-/** What the current model of each session can take, recorded by the last transform for the page tool. */-const supportBySession = new Map<string, ModelInputSupport>();+/**+ * The model each session's latest step used, recorded on every step (no+ * catalog read) for the page tool. The agent also calls that tool on PDFs it+ * finds in the workspace, where no attached PDF ever brings the session+ * through the PDF path; those calls got the text-only answer on image models.+ */+const modelBySession = new Map<string, { providerID: string; modelID: string }>();++function rememberModel(sessionID: string, providerID: string, modelID: string): void {+ // Re-inserting moves the session to the end: eviction takes the least+ // recently used, never a session that is still stepping.+ modelBySession.delete(sessionID);+ modelBySession.set(sessionID, { providerID, modelID });+ if (modelBySession.size > MAX_REMEMBERED_SESSIONS) {+ const oldest = modelBySession.keys().next().value;+ if (oldest !== undefined) modelBySession.delete(oldest);+ }+}function rememberDigest(root: string | null, source: PdfSource, digest: string): void {if (!source.key) return;@@ -456,7 +485,7 @@ async function inlineImages(root: string | null, derived: DerivedPdf, support: Mif (!support.image) return [];const stem = derived.filename.slice(0, -".pdf".length);const images: ImagePart[] = [];- let budget = INLINE_IMAGE_BUDGET_BYTES;+ let budget = support.imageBytes ?? INLINE_IMAGE_BUDGET_BYTES;for (const page of derived.renderedPages.slice(0, MAX_INLINE_PAGES)) {if (page.bytes > budget) break;const bytes = pageImageOf(derived, page);@@ -574,6 +603,31 @@ function stepModel(messages: unknown[]): StepModel | null {return null;}+/**+ * The step's model from the hook input. The 2.x engine bridge+ * (omnirush-engine2.ts) passes `{sessionID, model}` there and hands the hook+ * messages whose `info` is only `{id, role}`, so `stepModel` finds nothing in+ * them; the 1.x engine passes an empty input and full message infos.+ */+function hookModel(input: unknown): StepModel | null {+ if (!isRecord(input) || !isRecord(input.model)) return null;+ const providerID = optionalStringProperty(input.model, "providerID");+ const modelID = optionalStringProperty(input.model, "id") ?? optionalStringProperty(input.model, "modelID");+ if (!providerID || !modelID) return null;+ return { providerID, modelID, sessionID: optionalStringProperty(input, "sessionID") ?? null };+}++/**+ * What this route can take, beside what the model can: an omnirush.ai model+ * gets a PDF as its text and page images, never the file itself (native PDF+ * input is not verified through the gateway, and a PDF's base64 counts+ * against its request limit in every step), within the gateway's budget.+ */+function supportForRoute(providerID: string, support: ModelInputSupport): ModelInputSupport {+ if (providerID !== OMNIRUSH_PROVIDER_ID) return support;+ return { ...support, pdf: false, imageBytes: OMNIRUSH_IMAGE_BUDGET_BYTES };+}+function hasPdf(messages: unknown[]): boolean {return messages.some((message) => {if (!isRecord(message)) return false;@@ -616,15 +670,16 @@ export const OmniRushPdfAttachments = async (factoryInput?: unknown) => {: { resolve: async () => TEXT_ONLY };return {"experimental.chat.messages.transform": async (input: unknown, output: { messages: unknown[] }) => {- void input;+ const model = hookModel(input) ?? stepModel(output.messages);+ // Every agent step records its model, PDF or not (no catalog read): the+ // page tool needs it for PDFs the agent finds in the workspace itself.+ const agent = isRecord(input) ? optionalStringProperty(input, "agent") : undefined;+ if (model?.sessionID && !(agent && SIDE_AGENTS.has(agent))) rememberModel(model.sessionID, model.providerID, model.modelID);if (!hasPdf(output.messages)) return;const root = workspaceRoot(factoryContext);- const model = stepModel(output.messages);- const support = model ? await resolver.resolve(model.providerID, model.modelID) : TEXT_ONLY;- if (model?.sessionID) {- if (supportBySession.size > 256) supportBySession.clear();- supportBySession.set(model.sessionID, support);- }+ const support = model+ ? supportForRoute(model.providerID, await resolver.resolve(model.providerID, model.modelID))+ : TEXT_ONLY;const policy = support.pdf ? nativePdfPolicy(support.npm, support.contextTokens) : null;const nativeBudget: NativeBudget | null = policy? {@@ -649,14 +704,20 @@ export const OmniRushPdfAttachments = async (factoryInput?: unknown) => {async execute(args: { pdf_path: string; pages: number[] }, context: unknown) {const root = workspaceRoot(factoryContext);const sessionID = optionalStringProperty(context, "sessionID");- const support = (sessionID ? supportBySession.get(sessionID) : undefined) ?? TEXT_ONLY;+ const model = sessionID ? modelBySession.get(sessionID) : undefined;+ const support = model+ ? supportForRoute(model.providerID, await resolver.resolve(model.providerID, model.modelID))+ : TEXT_ONLY;const bytes = await readWorkspacePdf(root, args.pdf_path);const derived = await derivePdf(root, basename(args.pdf_path), bytes, { renderPages: false });if (derived.loadError) return `PDF could not be prepared: ${derived.loadError}`;if (!support.image) return pageToolOutput(derived, args.pages, [], support);const updated = await renderPdfPages(root, derived, bytes, args.pages);const wanted = new Set(args.pages);- const served = updated.renderedPages.filter((image) => wanted.has(image.page)).slice(0, MAX_PAGES_PER_REQUEST);+ const served = withinImageBudget(+ updated.renderedPages.filter((image) => wanted.has(image.page)).slice(0, MAX_PAGES_PER_REQUEST),+ support.imageBytes,+ );const attachments: Array<{ type: "file"; mime: string; url: string; filename?: string }> = [];for (const image of served) {const data = pageImageOf(updated, image);diff --git a/apps/server/src/pdf-attachments/capabilities.ts b/apps/server/src/pdf-attachments/capabilities.tsindex 25aa6c68..f5268666 100644--- a/apps/server/src/pdf-attachments/capabilities.ts+++ b/apps/server/src/pdf-attachments/capabilities.ts@@ -16,6 +16,12 @@ export type ModelInputSupport = {npm: string | null;/** Context window in tokens, when the catalog says. */contextTokens: number | null;+ /**+ * Page-image bytes (raw, before base64) one PDF or one page-tool call may+ * attach, when the route caps request bodies below the provider's own+ * limits; unset: the plugin's default budget.+ */+ imageBytes?: number;};/**@@ -79,6 +85,19 @@ export function nativePdfPolicy(npm: string | null, contextTokens: number | nullreturn base;}+/** The page images in order while their raw bytes fit `limit` (all of them without one). */+export function withinImageBudget<T extends { bytes: number }>(images: T[], limit: number | undefined): T[] {+ if (limit === undefined) return images;+ const kept: T[] = [];+ let left = limit;+ for (const image of images) {+ if (image.bytes > left) break;+ left -= image.bytes;+ kept.push(image);+ }+ return kept;+}+/** Bytes a binary occupies once base64-encoded into a request body. */export function encodedSize(rawBytes: number): number {return Math.ceil(rawBytes / 3) * 4;