Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix(broker): an attachment the model refuses is left out and the request sent again, so the session recovers

ogsamrat8555385d42b7
diff --git a/apps/server/src/omnirush-gateway-broker.ts b/apps/server/src/omnirush-gateway-broker.ts
index ffc39fbc..519792d7 100644
--- a/apps/server/src/omnirush-gateway-broker.ts
+++ b/apps/server/src/omnirush-gateway-broker.ts
@@ -11,6 +11,11 @@ import {
SUBAGENT_ROOT_SESSION_HEADER,
} from "./omnirush-swarm.js";
import { builtinOmniRushModelCatalog, omnirushModelWantsReasoningSummary } from "./omnirush-model-catalog.js";
+import { RejectedAttachments, isRejectedAttachmentError, type Attachment } from "./rejected-attachments.js";
+
+/** A refused attachment is left out and the model request sent again, at most this often per request. */
+const MAX_ATTACHMENT_RETRIES = 2;
+const MAX_ATTACHMENT_ERROR_BYTES = 256 * 1024;
const TRACE_CAPABILITY_TTL_MS = 5 * 60_000;
@@ -929,6 +934,12 @@ export class OmniRushGatewayBroker {
private refreshInFlight: Promise<boolean> | null = null;
/** Whether the last failed refresh left the session as it was (unreachable, 5xx, contended). */
private lastRefreshTransient = false;
+ /**
+ * Attachments the model refused (a corrupt PDF the read tool returned, an
+ * image that is not one): replaced by "[file could not be read: <name>]" in
+ * every later request, so one bad file cannot fail the rest of a session.
+ */
+ private readonly rejectedAttachments = new RejectedAttachments();
private capabilityCache: { gatewayUrl: string; accountKey: string; version: 2 | 3; expiresAt: number } | null = null;
private capabilityProbe: { key: string; promise: Promise<2 | 3> } | null = null;
@@ -1021,7 +1032,9 @@ export class OmniRushGatewayBroker {
// A picked sub-agent model the gateway refused moments ago is not tried
// again for every step of a running sub-agent: it goes to the main model.
const skipped = requestBody === undefined ? null : this.skipRefusedSubagentModel(request, normalizedPath, requestBody);
- const body = skipped?.body ?? requestBody;
+ let body = skipped?.body ?? requestBody;
+ const attachments = this.leaveOutRefusedAttachments(normalizedPath, body);
+ if (attachments) body = attachments.body;
let spent = this.state.accessToken;
let response = await this.forwardModelRequest(request, normalizedPath, body, spent);
// Two rounds at most: the first may only adopt a pair the desktop rotated,
@@ -1057,6 +1070,11 @@ export class OmniRushGatewayBroker {
},
}, { status: 401 });
}
+ if (attachments) {
+ const retried = await this.retryWithoutRefusedAttachments(request, normalizedPath, attachments, response);
+ response = retried.response;
+ body = retried.body;
+ }
if (skipped) {
this.reportSubagentFallback(request, { ...skipped.event, status: response.status, ok: response.ok });
} else if (!response.ok && response.status !== 401 && body !== undefined) {
@@ -1082,6 +1100,58 @@ export class OmniRushGatewayBroker {
});
}
+ /**
+ * A model request's attachments, with the ones the model refused earlier
+ * replaced; null for a request that is not JSON or carries none.
+ */
+ private leaveOutRefusedAttachments(path: string, body: ArrayBuffer | string | undefined): { parsed: unknown; attachments: Attachment[]; body: ArrayBuffer | string } | null {
+ if (body === undefined || (path !== "responses" && path !== "responses/compact")) return null;
+ let parsed: unknown;
+ try {
+ parsed = JSON.parse(typeof body === "string" ? body : new TextDecoder().decode(body));
+ } catch {
+ return null;
+ }
+ if (!parsed || typeof parsed !== "object") return null;
+ const prepared = this.rejectedAttachments.prepare(parsed);
+ if (prepared.attachments.length === 0) return null;
+ if (prepared.replaced.length > 0) {
+ this.log?.("info", "omnirush.ai: left out attachments the model refused earlier", { count: prepared.replaced.length });
+ return { parsed, attachments: prepared.attachments, body: JSON.stringify(prepared.body) };
+ }
+ return { parsed, attachments: prepared.attachments, body };
+ }
+
+ /**
+ * The model refused an attachment (400 "The file you uploaded is badly
+ * formatted or corrupted"): the suspects are left out and the request sent
+ * again, so the turn goes on with a "[file could not be read: x]" note. An
+ * answer the model accepted marks its attachments as good.
+ */
+ private async retryWithoutRefusedAttachments(
+ request: Request,
+ path: string,
+ state: { parsed: unknown; attachments: Attachment[]; body: ArrayBuffer | string },
+ first: Response,
+ ): Promise<{ response: Response; body: ArrayBuffer | string }> {
+ let response = first;
+ let { attachments, body } = state;
+ for (let round = 0; response.status === 400 && round < MAX_ATTACHMENT_RETRIES && this.state; round += 1) {
+ const raw = new Uint8Array(await response.arrayBuffer().catch(() => new ArrayBuffer(0)));
+ const kept = new Response(raw.byteLength ? raw : null, { status: response.status, statusText: response.statusText, headers: response.headers });
+ const message = new TextDecoder().decode(raw.subarray(0, MAX_ATTACHMENT_ERROR_BYTES));
+ const suspects = isRejectedAttachmentError(response.status, message) ? this.rejectedAttachments.reject(attachments, message) : [];
+ if (suspects.length === 0) return { response: kept, body };
+ const prepared = this.rejectedAttachments.prepare(state.parsed);
+ attachments = prepared.attachments;
+ body = JSON.stringify(prepared.body);
+ this.log?.("warn", "omnirush.ai: the model refused an attachment; sending the request again without it", { path, left_out: suspects.length });
+ response = await this.forwardModelRequest(request, path, body, this.state.accessToken);
+ }
+ if (response.ok) this.rejectedAttachments.accept(attachments.filter((item) => !this.rejectedAttachments.isBad(item.fingerprint)));
+ return { response, body };
+ }
+
/**
* One session uploader envelope. The deadline grows with the envelope's size
* (session-upload-budget.ts), and the retry with a refreshed bearer gets a
diff --git a/apps/server/src/rejected-attachments.test.ts b/apps/server/src/rejected-attachments.test.ts
new file mode 100644
index 00000000..e935e1f9
--- /dev/null
+++ b/apps/server/src/rejected-attachments.test.ts
@@ -0,0 +1,106 @@
+import { describe, expect, test } from "bun:test";
+
+import { OmniRushGatewayBroker } from "./omnirush-gateway-broker.js";
+import { RejectedAttachments, attachmentsIn, isRejectedAttachmentError, replaceAttachments } from "./rejected-attachments.js";
+
+// The 400 a tester got after the read tool returned an invalid PDF (every later turn failed with it).
+const CORRUPT = "The file you uploaded is badly formatted or corrupted. Please fix the file and try again.";
+const BAD_PDF = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyA+PgplbmRvYmoKdHJhaWxlcgo8PCAvUm9vdCAxIDAgUiA+PgolJUVPRgo=";
+const IMAGE = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==";
+
+function responsesBody(prompt: string, withPdf = true) {
+ return {
+ model: "gpt-6-sol",
+ stream: true,
+ input: [
+ { role: "user", content: [{ type: "input_text", text: "Open bad.pdf with your file read tool" }] },
+ { type: "function_call", call_id: "call_1", name: "read", arguments: "{\"filePath\":\"bad.pdf\"}" },
+ { type: "function_call_output", call_id: "call_1", output: "PDF read successfully" },
+ ...(withPdf ? [{ role: "user", content: [{ type: "input_file", filename: "bad.pdf", file_data: BAD_PDF }, { type: "input_image", image_url: IMAGE }] }] : []),
+ { role: "user", content: [{ type: "input_text", text: prompt }] },
+ ],
+ };
+}
+
+describe("rejected attachments", () => {
+ test("a 400 about a file or image is recognised; other errors are not", () => {
+ expect(isRejectedAttachmentError(400, CORRUPT)).toBe(true);
+ expect(isRejectedAttachmentError(400, "Invalid image data")).toBe(true);
+ expect(isRejectedAttachmentError(400, "Unsupported parameter: 'max_output_tokens'")).toBe(false);
+ expect(isRejectedAttachmentError(500, CORRUPT)).toBe(false);
+ });
+
+ test("a new PDF and a new image refused for \"the file\": only the PDF is suspected", () => {
+ const both = attachmentsIn(responsesBody("x"));
+ expect(new RejectedAttachments().reject(both, CORRUPT).map((item) => item.name)).toEqual(["bad.pdf"]);
+ expect(new RejectedAttachments().reject(both, "Invalid image data").map((item) => item.name)).toEqual(["image"]);
+ expect(new RejectedAttachments().reject(both, "Invalid content").map((item) => item.name)).toEqual(["bad.pdf", "image"]);
+ });
+
+ test("the suspects are the attachments no accepted request carried; they are replaced from then on", () => {
+ const guard = new RejectedAttachments();
+ guard.accept(attachmentsIn({ input: [{ type: "input_image", image_url: IMAGE }] }));
+ const prepared = guard.prepare(responsesBody("two"));
+ expect(guard.reject(prepared.attachments).map((item) => item.name)).toEqual(["bad.pdf"]);
+ const next = guard.prepare(responsesBody("three"));
+ expect(next.replaced).toEqual(["bad.pdf"]);
+ expect(JSON.stringify(next.body)).toContain("[file could not be read: bad.pdf]");
+ expect(JSON.stringify(next.body)).toContain(IMAGE);
+ expect(replaceAttachments(next.body, () => false).body).toBe(next.body);
+ });
+
+ test("the broker sends the refused request again without the file, and leaves it out of later turns", async () => {
+ const sent: string[] = [];
+ const broker = new OmniRushGatewayBroker({
+ credentials: { gatewayUrl: "https://gateway.example/omnirush/v1", accessToken: "access-token", refreshToken: "refresh-token" },
+ engineToken: "local-engine-token",
+ fetch: async (_input, init) => {
+ const raw = init?.body;
+ const text = typeof raw === "string" ? raw : new TextDecoder().decode(raw as ArrayBuffer);
+ sent.push(text);
+ if (text.includes("data:application/pdf")) {
+ return new Response(JSON.stringify({ error: { message: CORRUPT, type: "invalid_request_error" } }), { status: 400, headers: { "content-type": "text/event-stream" } });
+ }
+ return new Response("data: ok\n\n", { headers: { "content-type": "text/event-stream" } });
+ },
+ });
+ const turn = (prompt: string) => broker.handle(new Request("http://127.0.0.1/omnirush-gateway/v1/responses", {
+ method: "POST",
+ headers: { Authorization: "Bearer local-engine-token", "Content-Type": "application/json" },
+ body: JSON.stringify(responsesBody(prompt)),
+ }), "responses");
+
+ const first = await turn("what did you get?");
+ expect(first.status).toBe(200);
+ expect(await first.text()).toContain("data: ok");
+ expect(sent.length).toBe(2);
+ expect(sent[1]).toContain("[file could not be read: bad.pdf]");
+ expect(sent[1]).not.toContain("data:application/pdf");
+ expect(sent[1]).toContain(IMAGE);
+
+ const second = await turn("Create ok.txt containing ok");
+ expect(second.status).toBe(200);
+ await second.text();
+ expect(sent.length).toBe(3);
+ expect(sent[2]).not.toContain("data:application/pdf");
+ });
+
+ test("a 400 that is not about an attachment passes through once", async () => {
+ let calls = 0;
+ const broker = new OmniRushGatewayBroker({
+ credentials: { gatewayUrl: "https://gateway.example/omnirush/v1", accessToken: "access-token", refreshToken: "refresh-token" },
+ engineToken: "local-engine-token",
+ fetch: async () => {
+ calls += 1;
+ return Response.json({ error: { message: "Unsupported parameter: 'max_output_tokens'" } }, { status: 400 });
+ },
+ });
+ const response = await broker.handle(new Request("http://127.0.0.1/omnirush-gateway/v1/responses", {
+ method: "POST",
+ headers: { Authorization: "Bearer local-engine-token", "Content-Type": "application/json" },
+ body: JSON.stringify(responsesBody("x")),
+ }), "responses");
+ expect(response.status).toBe(400);
+ expect(calls).toBe(1);
+ });
+});
diff --git a/apps/server/src/rejected-attachments.ts b/apps/server/src/rejected-attachments.ts
new file mode 100644
index 00000000..1eb26cf2
--- /dev/null
+++ b/apps/server/src/rejected-attachments.ts
@@ -0,0 +1,230 @@
+// An attachment the model API refuses (a corrupt PDF the read tool returned,
+// an image that is not one) stays in the conversation, so every later request
+// carries it again and fails with the same 400: one bad file ends the session.
+//
+// RejectedAttachments remembers the attachments the model refused (by the
+// SHA-256 of their data) and replaces them, in every later request, with a
+// short text part: `[file could not be read: <name>]`. Which part was refused
+// is not named by the API, so the suspects of a refusal are the request's
+// attachments that no earlier accepted request carried (all of them when every
+// one was accepted before). The gateway broker (omnirush-gateway-broker.ts)
+// retries the failing request at once without the part. Kept in step with the
+// CLI's copy (omnirush-cli assets/extensions/omnirush/rejected-attachments.js).
+//
+// Request shapes understood (anywhere in the JSON body):
+// Responses API { type: "input_file", filename, file_data | file_url | file_id }
+// { type: "input_image", image_url | file_id }
+// Chat Completions { type: "image_url", image_url: { url } }
+// { type: "file", file: { filename, file_data | file_id } }
+// Messages / pi { type: "image" | "document", source: { data | url } }
+// { type: "image", data, mimeType }
+
+import { createHash } from "node:crypto";
+
+const MAX_WALK_DEPTH = 64;
+
+/**
+ * Whether a model API error says an attachment could not be used: a 400
+ * whose message names a file/image/PDF and calls it invalid, corrupt,
+ * unreadable or unsupported.
+ */
+export function isRejectedAttachmentError(status: unknown, message: unknown): boolean {
+ if (Number(status) !== 400) return false;
+ const text = String(message ?? "").slice(0, 8192);
+ if (!/\b(?:file|files|image|images|pdf|attachment|document|media|input_file|input_image|image_url|file_data)\b/i.test(text)) return false;
+ return /invalid|corrupt|badly formatted|malformed|could not (?:be )?(?:process|read|decod|pars|open|load)|unable to (?:process|read|decod|pars|open|load)|failed to (?:process|read|decod|pars|open|load)|not a valid|unsupported|is not supported|cannot be (?:processed|read|decoded|parsed)|did not (?:contain|look like)|no pages/i.test(text);
+}
+
+function isRecord(value: unknown): value is Record<string, any> {
+ return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+
+function dataUrlName(url: unknown): string | null {
+ const mime = /^data:([^;,]+)/i.exec(String(url))?.[1];
+ if (!mime) return null;
+ return mime.startsWith("image/") ? "image" : mime === "application/pdf" ? "document.pdf" : "attachment";
+}
+
+export type AttachmentStyle = "input_text" | "text";
+export type AttachmentKind = "image" | "file";
+export type Attachment = { fingerprint: string; name: string; kind: AttachmentKind };
+
+function kindOf(data: string): AttachmentKind {
+ return /^data:image\//i.test(data) ? "image" : "file";
+}
+
+/**
+ * Which kind of attachment an error message blames: "image" when it names an
+ * image and no file/PDF/document, "file" for the reverse, else null (either).
+ */
+export function blamedKind(message: unknown): AttachmentKind | null {
+ const text = String(message ?? "");
+ const image = /\bimages?\b|image_url|input_image/i.test(text);
+ const file = /\b(?:files?|pdfs?|documents?)\b|input_file|file_data/i.test(text);
+ if (image && !file) return "image";
+ if (file && !image) return "file";
+ return null;
+}
+
+/** The attachment a content part carries ({ data, name, style, kind }), or null. */
+export function attachmentOf(part: unknown): { data: string; name: string; style: AttachmentStyle; kind: AttachmentKind } | null {
+ if (!isRecord(part) || typeof part.type !== "string") return null;
+ const str = (value: unknown): string | null => (typeof value === "string" && value.length > 0 ? value : null);
+ switch (part.type) {
+ case "input_file": {
+ const data = str(part.file_data) ?? str(part.file_url) ?? str(part.file_id);
+ return data ? { data, name: str(part.filename) ?? dataUrlName(data) ?? "file", style: "input_text", kind: kindOf(data) } : null;
+ }
+ case "input_image": {
+ const url = isRecord(part.image_url) ? part.image_url.url : part.image_url;
+ const data = str(url) ?? str(part.file_id);
+ return data ? { data, name: str(part.filename) ?? "image", style: "input_text", kind: "image" } : null;
+ }
+ case "image_url": {
+ const url = isRecord(part.image_url) ? part.image_url.url : part.image_url;
+ const data = str(url);
+ return data ? { data, name: "image", style: "text", kind: "image" } : null;
+ }
+ case "file": {
+ const file = isRecord(part.file) ? part.file : null;
+ const data = file ? str(file.file_data) ?? str(file.file_id) ?? str(file.url) : null;
+ return data && file ? { data, name: str(file.filename) ?? dataUrlName(data) ?? "file", style: "text", kind: kindOf(data) } : null;
+ }
+ case "image":
+ case "document": {
+ const source = isRecord(part.source) ? part.source : null;
+ const data = str(part.data) ?? (source ? str(source.data) ?? str(source.url) ?? str(source.file_id) : null);
+ const name = str(part.filename) ?? str(part.name) ?? str(part.title) ?? (part.type === "image" ? "image" : "document");
+ return data ? { data, name, style: "text", kind: part.type === "image" ? "image" : kindOf(data) } : null;
+ }
+ default:
+ return null;
+ }
+}
+
+export function fingerprint(data: string): string {
+ return createHash("sha256").update(String(data)).digest("hex");
+}
+
+/** The placeholder that stands in for a refused attachment. */
+export function placeholderText(name: string): string {
+ const clean = String(name || "file").replace(/[\r\n\]]+/g, " ").slice(0, 200);
+ return `[file could not be read: ${clean}]`;
+}
+
+/** Every attachment of a request body (or message list), in order: { fingerprint, name }. */
+export function attachmentsIn(body: unknown): Attachment[] {
+ const out: Attachment[] = [];
+ const walk = (value: unknown, depth: number): void => {
+ if (depth > MAX_WALK_DEPTH) return;
+ if (Array.isArray(value)) {
+ for (const item of value) walk(item, depth + 1);
+ return;
+ }
+ if (!isRecord(value)) return;
+ const attachment = attachmentOf(value);
+ if (attachment) {
+ out.push({ fingerprint: fingerprint(attachment.data), name: attachment.name, kind: attachment.kind });
+ return;
+ }
+ for (const key of Object.keys(value)) walk(value[key], depth + 1);
+ };
+ walk(body, 0);
+ return out;
+}
+
+/**
+ * A copy of `body` with every attachment whose fingerprint `isBad` accepts
+ * replaced by a text part of the same API style; `replaced` lists their names.
+ * The body is returned as is (same object) when nothing is replaced.
+ */
+export function replaceAttachments<T>(body: T, isBad: (fingerprint: string) => boolean): { body: T; replaced: string[] } {
+ const replaced: string[] = [];
+ const walk = (value: unknown, depth: number): unknown => {
+ if (depth > MAX_WALK_DEPTH) return value;
+ if (Array.isArray(value)) {
+ let changed = false;
+ const next = value.map((item) => {
+ const out = walk(item, depth + 1);
+ if (out !== item) changed = true;
+ return out;
+ });
+ return changed ? next : value;
+ }
+ if (!isRecord(value)) return value;
+ const attachment = attachmentOf(value);
+ if (attachment) {
+ if (!isBad(fingerprint(attachment.data))) return value;
+ replaced.push(attachment.name);
+ return { type: attachment.style, text: placeholderText(attachment.name) };
+ }
+ let next: Record<string, unknown> = value;
+ for (const key of Object.keys(value)) {
+ const out = walk(value[key], depth + 1);
+ if (out !== value[key]) {
+ if (next === value) next = { ...value };
+ next[key] = out;
+ }
+ }
+ return next;
+ };
+ const out = walk(body, 0) as T;
+ return { body: out, replaced };
+}
+
+export class RejectedAttachments {
+ /** Carried by a request the model accepted. */
+ private readonly accepted = new Set<string>();
+ /** fingerprint -> name: refused; replaced from now on. */
+ private readonly bad = new Map<string, string>();
+
+ constructor(private readonly maxEntries = 10_000) {}
+
+ get size(): number {
+ return this.bad.size;
+ }
+
+ isBad(fp: string): boolean {
+ return this.bad.has(fp);
+ }
+
+ /** The request's attachments (fingerprints) and its body with the refused ones replaced. */
+ prepare<T>(body: T): { body: T; attachments: Attachment[]; replaced: string[] } {
+ const attachments = attachmentsIn(body);
+ if (attachments.length === 0 || this.bad.size === 0) return { body, attachments, replaced: [] };
+ const { body: next, replaced } = replaceAttachments(body, (fp) => this.bad.has(fp));
+ return { body: next, attachments, replaced };
+ }
+
+ /** The model answered a request that carried these attachments. */
+ accept(attachments: readonly Attachment[]): void {
+ for (const { fingerprint: fp } of attachments) {
+ if (this.bad.has(fp)) continue;
+ if (this.accepted.size >= this.maxEntries) this.accepted.clear();
+ this.accepted.add(fp);
+ }
+ }
+
+ /**
+ * The model refused a request carrying these attachments for one of them:
+ * marks the suspects as refused and returns them ([] when there is nothing
+ * left to leave out). Suspects: the attachments not refused yet, narrowed to
+ * the ones no accepted request carried, then to the kind the error message
+ * blames (`message`: "The file you uploaded..." blames files, not images),
+ * each step only when it leaves some.
+ */
+ reject(attachments: readonly Attachment[], message = ""): Attachment[] {
+ const open = attachments.filter((item) => !this.bad.has(item.fingerprint));
+ const fresh = open.filter((item) => !this.accepted.has(item.fingerprint));
+ let suspects = fresh.length > 0 ? fresh : open;
+ const blamed = blamedKind(message);
+ const ofKind = blamed ? suspects.filter((item) => item.kind === blamed) : [];
+ if (ofKind.length > 0) suspects = ofKind;
+ for (const item of suspects) {
+ if (this.bad.size >= this.maxEntries) break;
+ this.bad.set(item.fingerprint, item.name);
+ this.accepted.delete(item.fingerprint);
+ }
+ return suspects;
+ }
+}