Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix(pdf): page images for PDFs the agent finds in the workspace (#60)

Skandesh1e328a8f2d5e
diff --git a/apps/server/src/opencode-plugins/omnirush-engine2.ts b/apps/server/src/opencode-plugins/omnirush-engine2.ts
index 94854c53..5c5f1856 100644
--- a/apps/server/src/opencode-plugins/omnirush-engine2.ts
+++ b/apps/server/src/opencode-plugins/omnirush-engine2.ts
@@ -369,7 +369,8 @@ export default {
if (messageHooks.length && Array.isArray(event.messages)) {
const { v1, restore } = messagesForV1(event.messages);
const output = { messages: v1 };
- for (const hook of messageHooks) await hook({ sessionID: input.sessionID, model: input.model }, output);
+ // `agent` names title and compaction requests, which may run on another model.
+ for (const hook of messageHooks) await hook({ sessionID: input.sessionID, model: input.model, agent: input.agent }, output);
event.messages = restore(output.messages as Rec[]);
}
if (paramHooks.length && isRec(event.options)) {
diff --git a/apps/server/src/opencode-plugins/omnirush-pdf-attachments.test.ts b/apps/server/src/opencode-plugins/omnirush-pdf-attachments.test.ts
index df318079..a3cac1b0 100644
--- a/apps/server/src/opencode-plugins/omnirush-pdf-attachments.test.ts
+++ b/apps/server/src/opencode-plugins/omnirush-pdf-attachments.test.ts
@@ -8,6 +8,7 @@ import { buildOmniRushRuntimeConfigObject } from "../omnirush-runtime-config.js"
import { omnirushPdfAttachmentsPluginPath } from "../omnirush-extensions-plugin-path.js";
import { resetDerivedPdfMemory } from "../pdf-attachments/derive.js";
import { buildTestPdf, corruptTestPdf, pdfDataUrl } from "../pdf-attachments/pdf-fixture.test-helper.js";
+import { withinImageBudget } from "../pdf-attachments/capabilities.js";
import { OmniRushPdfAttachments } from "./omnirush-pdf-attachments.js";
const PACKAGE_ROOT = join(dirname(fileURLToPath(import.meta.url)), "../..");
@@ -23,6 +24,7 @@ const catalog = {
{ id: "openrouter", npm: "@ai-sdk/openai-compatible", models: { vision: { id: "vision", attachment: true, modalities: { input: ["text", "image"], output: ["text"] } } } },
{ id: "ollama", npm: "@ai-sdk/openai-compatible", models: { text: { id: "text", attachment: false, modalities: { input: ["text"], output: ["text"] } } } },
{ id: "odd", npm: "@ai-sdk/openai-compatible", models: { "pdf-no-vision": { id: "pdf-no-vision", attachment: true, modalities: { input: ["text", "pdf"], output: ["text"] } } } },
+ { id: "omnirush", models: { "gpt-6-astra": { id: "gpt-6-astra", providerID: "omnirush", api: { id: "gpt-6-astra", url: "", npm: "@ai-sdk/openai" }, limit: { context: 400000, output: 128000 }, capabilities: { attachment: true, toolcall: true, input: { text: true, audio: false, image: true, video: false, pdf: true }, output: { text: true, audio: false, image: false, video: false, pdf: false } } } } },
],
default: {},
connected: [],
@@ -35,6 +37,7 @@ const NATIVE_100_PAGES: Model = { providerID: "openai", modelID: "gpt-native" };
const VISION: Model = { providerID: "openrouter", modelID: "vision" };
const TEXT: Model = { providerID: "ollama", modelID: "text" };
const UNLISTED: Model = { providerID: "custom", modelID: "mystery" };
+const ASTRA: Model = { providerID: "omnirush", modelID: "gpt-6-astra" };
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
@@ -461,6 +464,91 @@ describe("OmniRush.ai PDF attachments plugin", () => {
});
}, 30_000);
+ test("the page tool renders pages of a workspace PDF no chat message attached, for an image-capable session", async () => {
+ // Production 2026-09-30: agents open PDFs they find in the project with the
+ // page tool; with no PDF in the chat the session's model was never
+ // recorded, so image models got the text-only answer and no pages.
+ await withWorkspace(async (root) => {
+ await mkdir(join(root, "docs"), { recursive: true });
+ await writeFile(join(root, "docs", "spec.pdf"), buildTestPdf(["Spec page one", "Spec page two"]));
+ const plugin = await OmniRushPdfAttachments({ directory: root, client: { provider: { list: async () => catalog } } });
+ const tool = plugin.tool.omnirush_pdf_pages;
+
+ // Session ids no other test uses: the plugin's session memory is module-wide.
+ const visionStep = { messages: [userMessage(VISION, [question], "mw1")] };
+ visionStep.messages[0].info.sessionID = "ses-ws-vision";
+ await plugin["experimental.chat.messages.transform"]({}, visionStep);
+ const visionResult = await tool.execute({ pdf_path: "docs/spec.pdf", pages: [1, 2] }, { sessionID: "ses-ws-vision" });
+ if (typeof visionResult === "string") throw new Error(`Expected page images, got: ${visionResult}`);
+ expect(visionResult.attachments.map((attachment) => attachment.mime)).toEqual(["image/png", "image/png"]);
+ expect(visionResult.output).toContain("page_images_attached: pages 1-2, in order");
+
+ const textStep = { messages: [userMessage(TEXT, [question], "mw2")] };
+ textStep.messages[0].info.sessionID = "ses-ws-text";
+ await plugin["experimental.chat.messages.transform"]({}, textStep);
+ const textResult = await tool.execute({ pdf_path: "docs/spec.pdf", pages: [1] }, { sessionID: "ses-ws-text" });
+ expect(String(textResult)).toContain("page_images_attached: none (this model cannot view images; text is provided instead)");
+ });
+ }, 30_000);
+
+ test("on the 2.x engine the step's model comes from the hook input, since the bridge's messages carry no model", async () => {
+ // omnirush-engine2.ts messagesForV1 gives each message only {id, role} and
+ // passes {sessionID, model} as the hook input.
+ await withWorkspace(async (root) => {
+ await mkdir(join(root, "docs"), { recursive: true });
+ await writeFile(join(root, "docs", "spec.pdf"), buildTestPdf(["Spec page one", "Spec page two"]));
+ const plugin = await OmniRushPdfAttachments({ directory: root, client: { provider: { list: async () => catalog } } });
+ const bridged = { messages: [{ info: { id: "m0", role: "user" }, content: [{ type: "text", text: "Read docs/spec.pdf" }] }] };
+ await plugin["experimental.chat.messages.transform"]({ sessionID: "ses-v2", model: { id: "vision", providerID: "openrouter" }, agent: "build" }, bridged);
+ // A title request on a small text-only model must not replace the session's model.
+ await plugin["experimental.chat.messages.transform"]({ sessionID: "ses-v2", model: { id: "text", providerID: "ollama" }, agent: "title" }, structuredClone(bridged));
+ const result = await plugin.tool.omnirush_pdf_pages.execute({ pdf_path: "docs/spec.pdf", pages: [1, 2] }, { sessionID: "ses-v2" });
+ if (typeof result === "string") throw new Error(`Expected page images, got: ${result}`);
+ expect(result.attachments.map((attachment) => attachment.mime)).toEqual(["image/png", "image/png"]);
+ expect(result.output).toContain("page_images_attached: pages 1-2, in order");
+ });
+ }, 30_000);
+
+ test("an omnirush.ai model gets a PDF as page images and text, never the PDF file itself", async () => {
+ await withWorkspace(async (root) => {
+ const pdf = buildTestPdf(["Quarterly revenue report", "Appendix: totals"]);
+ const step = { messages: [userMessage(ASTRA, [pdfPart(pdfDataUrl(pdf)), question], "ma1")] };
+ step.messages[0].info.sessionID = "ses-astra";
+ const [message] = await transform(root, step.messages);
+ const parts = partsOf(message);
+ expect(parts.some((part) => part.mime === "application/pdf")).toBe(false);
+ expect(parts.filter((part) => part.type === "file" && part.mime === "image/png").length).toBe(2);
+ expect(noteOf(message)).toContain("page_images_in_this_message: pages 1-2, in order");
+ });
+ }, 30_000);
+
+ test("a session that keeps stepping is never evicted from the plugin's session memory", async () => {
+ await withWorkspace(async (root) => {
+ await mkdir(join(root, "docs"), { recursive: true });
+ await writeFile(join(root, "docs", "spec.pdf"), buildTestPdf(["Spec page one"]));
+ const plugin = await OmniRushPdfAttachments({ directory: root, client: { provider: { list: async () => catalog } } });
+ const step = async (sessionID: string) => {
+ const output = { messages: [userMessage(VISION, [question], `m-${sessionID}`)] };
+ output.messages[0].info.sessionID = sessionID;
+ await plugin["experimental.chat.messages.transform"]({}, output);
+ };
+ await step("ses-busy");
+ for (let index = 0; index < 200; index += 1) await step(`ses-other-${index}`);
+ await step("ses-busy");
+ for (let index = 200; index < 300; index += 1) await step(`ses-other-${index}`);
+ const result = await plugin.tool.omnirush_pdf_pages.execute({ pdf_path: "docs/spec.pdf", pages: [1] }, { sessionID: "ses-busy" });
+ if (typeof result === "string") throw new Error(`Expected page images, got: ${result}`);
+ expect(result.output).toContain("page_images_attached: pages 1, in order");
+ });
+ }, 30_000);
+
+ test("page images stop at the route's byte budget, in page order", () => {
+ const pages = [{ page: 1, bytes: 800 }, { page: 2, bytes: 700 }, { page: 3, bytes: 900 }];
+ expect(withinImageBudget(pages, undefined).map((image) => image.page)).toEqual([1, 2, 3]);
+ expect(withinImageBudget(pages, 1_600).map((image) => image.page)).toEqual([1, 2]);
+ expect(withinImageBudget(pages, 500)).toEqual([]);
+ });
+
test("is registered in runtime config, bundled with its wasm runtime, and packaged by the desktop app", async () => {
const runtime = await buildOmniRushRuntimeConfigObject();
const plugin = runtime.plugin;
diff --git a/apps/server/src/opencode-plugins/omnirush-pdf-attachments.ts b/apps/server/src/opencode-plugins/omnirush-pdf-attachments.ts
index b142e1b9..9cc037f3 100644
--- a/apps/server/src/opencode-plugins/omnirush-pdf-attachments.ts
+++ b/apps/server/src/opencode-plugins/omnirush-pdf-attachments.ts
@@ -2,7 +2,7 @@ import { realpath } from "node:fs/promises";
import { basename, isAbsolute, relative, resolve } from "node:path";
import { fileURLToPath } from "node:url";
import { z } from "zod";
-import { createInputSupportResolver, encodedSize, nativePdfPolicy, TEXT_ONLY } from "../pdf-attachments/capabilities.js";
+import { createInputSupportResolver, encodedSize, nativePdfPolicy, TEXT_ONLY, withinImageBudget } from "../pdf-attachments/capabilities.js";
import type { InputSupportResolver, ModelInputSupport, NativePdfPolicy } from "../pdf-attachments/capabilities.js";
import {
cachedDerivedPdf,
@@ -45,6 +45,19 @@ const MAX_PDF_BYTES = 64 * MIB;
const MAX_INLINE_PAGES = EAGER_RENDERED_PAGES;
/** Total inline image bytes per PDF; keeps requests under provider payload limits. */
const INLINE_IMAGE_BUDGET_BYTES = 12 * MIB;
+/** The omnirush.ai models (omnirush-runtime-config's internal provider). */
+const OMNIRUSH_PROVIDER_ID = "omnirush";
+/**
+ * Page-image bytes per PDF (and per page-tool call) for an omnirush.ai model.
+ * The omnirush.ai gateway refuses a request body over 8 MiB
+ * (`model_request_too_large`) and images ride base64-encoded in every later
+ * step too, so 2 MiB (about 2.7 MiB encoded) leaves room for the rest.
+ */
+const OMNIRUSH_IMAGE_BUDGET_BYTES = 2 * MIB;
+/** Sessions whose model the plugin remembers; the least recently used goes first. */
+const MAX_REMEMBERED_SESSIONS = 256;
+/** Side requests that may run on another (small) model; their model is not the session's. */
+const SIDE_AGENTS = new Set(["title", "compaction", "summary"]);
/** Extracted text inlined in the note; the full text stays on disk. */
const MAX_INLINE_TEXT_CHARS = 60_000;
const PAGE_TOOL_NAME = "omnirush_pdf_pages";
@@ -364,8 +377,24 @@ const digestByKey = new Map<string, string>();
function scopedMemoKey(root: string | null, key: string): string {
return `${root ?? ""}\n${key}`;
}
-/** What the current model of each session can take, recorded by the last transform for the page tool. */
-const supportBySession = new Map<string, ModelInputSupport>();
+/**
+ * The model each session's latest step used, recorded on every step (no
+ * catalog read) for the page tool. The agent also calls that tool on PDFs it
+ * finds in the workspace, where no attached PDF ever brings the session
+ * through the PDF path; those calls got the text-only answer on image models.
+ */
+const modelBySession = new Map<string, { providerID: string; modelID: string }>();
+
+function rememberModel(sessionID: string, providerID: string, modelID: string): void {
+ // Re-inserting moves the session to the end: eviction takes the least
+ // recently used, never a session that is still stepping.
+ modelBySession.delete(sessionID);
+ modelBySession.set(sessionID, { providerID, modelID });
+ if (modelBySession.size > MAX_REMEMBERED_SESSIONS) {
+ const oldest = modelBySession.keys().next().value;
+ if (oldest !== undefined) modelBySession.delete(oldest);
+ }
+}
function rememberDigest(root: string | null, source: PdfSource, digest: string): void {
if (!source.key) return;
@@ -456,7 +485,7 @@ async function inlineImages(root: string | null, derived: DerivedPdf, support: M
if (!support.image) return [];
const stem = derived.filename.slice(0, -".pdf".length);
const images: ImagePart[] = [];
- let budget = INLINE_IMAGE_BUDGET_BYTES;
+ let budget = support.imageBytes ?? INLINE_IMAGE_BUDGET_BYTES;
for (const page of derived.renderedPages.slice(0, MAX_INLINE_PAGES)) {
if (page.bytes > budget) break;
const bytes = pageImageOf(derived, page);
@@ -574,6 +603,31 @@ function stepModel(messages: unknown[]): StepModel | null {
return null;
}
+/**
+ * The step's model from the hook input. The 2.x engine bridge
+ * (omnirush-engine2.ts) passes `{sessionID, model}` there and hands the hook
+ * messages whose `info` is only `{id, role}`, so `stepModel` finds nothing in
+ * them; the 1.x engine passes an empty input and full message infos.
+ */
+function hookModel(input: unknown): StepModel | null {
+ if (!isRecord(input) || !isRecord(input.model)) return null;
+ const providerID = optionalStringProperty(input.model, "providerID");
+ const modelID = optionalStringProperty(input.model, "id") ?? optionalStringProperty(input.model, "modelID");
+ if (!providerID || !modelID) return null;
+ return { providerID, modelID, sessionID: optionalStringProperty(input, "sessionID") ?? null };
+}
+
+/**
+ * What this route can take, beside what the model can: an omnirush.ai model
+ * gets a PDF as its text and page images, never the file itself (native PDF
+ * input is not verified through the gateway, and a PDF's base64 counts
+ * against its request limit in every step), within the gateway's budget.
+ */
+function supportForRoute(providerID: string, support: ModelInputSupport): ModelInputSupport {
+ if (providerID !== OMNIRUSH_PROVIDER_ID) return support;
+ return { ...support, pdf: false, imageBytes: OMNIRUSH_IMAGE_BUDGET_BYTES };
+}
+
function hasPdf(messages: unknown[]): boolean {
return messages.some((message) => {
if (!isRecord(message)) return false;
@@ -616,15 +670,16 @@ export const OmniRushPdfAttachments = async (factoryInput?: unknown) => {
: { resolve: async () => TEXT_ONLY };
return {
"experimental.chat.messages.transform": async (input: unknown, output: { messages: unknown[] }) => {
- void input;
+ const model = hookModel(input) ?? stepModel(output.messages);
+ // Every agent step records its model, PDF or not (no catalog read): the
+ // page tool needs it for PDFs the agent finds in the workspace itself.
+ const agent = isRecord(input) ? optionalStringProperty(input, "agent") : undefined;
+ if (model?.sessionID && !(agent && SIDE_AGENTS.has(agent))) rememberModel(model.sessionID, model.providerID, model.modelID);
if (!hasPdf(output.messages)) return;
const root = workspaceRoot(factoryContext);
- const model = stepModel(output.messages);
- const support = model ? await resolver.resolve(model.providerID, model.modelID) : TEXT_ONLY;
- if (model?.sessionID) {
- if (supportBySession.size > 256) supportBySession.clear();
- supportBySession.set(model.sessionID, support);
- }
+ const support = model
+ ? supportForRoute(model.providerID, await resolver.resolve(model.providerID, model.modelID))
+ : TEXT_ONLY;
const policy = support.pdf ? nativePdfPolicy(support.npm, support.contextTokens) : null;
const nativeBudget: NativeBudget | null = policy
? {
@@ -649,14 +704,20 @@ export const OmniRushPdfAttachments = async (factoryInput?: unknown) => {
async execute(args: { pdf_path: string; pages: number[] }, context: unknown) {
const root = workspaceRoot(factoryContext);
const sessionID = optionalStringProperty(context, "sessionID");
- const support = (sessionID ? supportBySession.get(sessionID) : undefined) ?? TEXT_ONLY;
+ const model = sessionID ? modelBySession.get(sessionID) : undefined;
+ const support = model
+ ? supportForRoute(model.providerID, await resolver.resolve(model.providerID, model.modelID))
+ : TEXT_ONLY;
const bytes = await readWorkspacePdf(root, args.pdf_path);
const derived = await derivePdf(root, basename(args.pdf_path), bytes, { renderPages: false });
if (derived.loadError) return `PDF could not be prepared: ${derived.loadError}`;
if (!support.image) return pageToolOutput(derived, args.pages, [], support);
const updated = await renderPdfPages(root, derived, bytes, args.pages);
const wanted = new Set(args.pages);
- const served = updated.renderedPages.filter((image) => wanted.has(image.page)).slice(0, MAX_PAGES_PER_REQUEST);
+ const served = withinImageBudget(
+ updated.renderedPages.filter((image) => wanted.has(image.page)).slice(0, MAX_PAGES_PER_REQUEST),
+ support.imageBytes,
+ );
const attachments: Array<{ type: "file"; mime: string; url: string; filename?: string }> = [];
for (const image of served) {
const data = pageImageOf(updated, image);
diff --git a/apps/server/src/pdf-attachments/capabilities.ts b/apps/server/src/pdf-attachments/capabilities.ts
index 25aa6c68..f5268666 100644
--- a/apps/server/src/pdf-attachments/capabilities.ts
+++ b/apps/server/src/pdf-attachments/capabilities.ts
@@ -16,6 +16,12 @@ export type ModelInputSupport = {
npm: string | null;
/** Context window in tokens, when the catalog says. */
contextTokens: number | null;
+ /**
+ * Page-image bytes (raw, before base64) one PDF or one page-tool call may
+ * attach, when the route caps request bodies below the provider's own
+ * limits; unset: the plugin's default budget.
+ */
+ imageBytes?: number;
};
/**
@@ -79,6 +85,19 @@ export function nativePdfPolicy(npm: string | null, contextTokens: number | null
return base;
}
+/** The page images in order while their raw bytes fit `limit` (all of them without one). */
+export function withinImageBudget<T extends { bytes: number }>(images: T[], limit: number | undefined): T[] {
+ if (limit === undefined) return images;
+ const kept: T[] = [];
+ let left = limit;
+ for (const image of images) {
+ if (image.bytes > left) break;
+ left -= image.bytes;
+ kept.push(image);
+ }
+ return kept;
+}
+
/** Bytes a binary occupies once base64-encoded into a request body. */
export function encodedSize(rawBytes: number): number {
return Math.ceil(rawBytes / 3) * 4;