A desktop coding agent with free access to frontier models.
TypeScript
fix(broker): an attachment the model refuses is left out and the request sent again, so the session recovers
ogsamrat8555385d42b7
diff --git a/apps/server/src/omnirush-gateway-broker.ts b/apps/server/src/omnirush-gateway-broker.tsindex ffc39fbc..519792d7 100644--- a/apps/server/src/omnirush-gateway-broker.ts+++ b/apps/server/src/omnirush-gateway-broker.ts@@ -11,6 +11,11 @@ import {SUBAGENT_ROOT_SESSION_HEADER,} from "./omnirush-swarm.js";import { builtinOmniRushModelCatalog, omnirushModelWantsReasoningSummary } from "./omnirush-model-catalog.js";+import { RejectedAttachments, isRejectedAttachmentError, type Attachment } from "./rejected-attachments.js";++/** A refused attachment is left out and the model request sent again, at most this often per request. */+const MAX_ATTACHMENT_RETRIES = 2;+const MAX_ATTACHMENT_ERROR_BYTES = 256 * 1024;const TRACE_CAPABILITY_TTL_MS = 5 * 60_000;@@ -929,6 +934,12 @@ export class OmniRushGatewayBroker {private refreshInFlight: Promise<boolean> | null = null;/** Whether the last failed refresh left the session as it was (unreachable, 5xx, contended). */private lastRefreshTransient = false;+ /**+ * Attachments the model refused (a corrupt PDF the read tool returned, an+ * image that is not one): replaced by "[file could not be read: <name>]" in+ * every later request, so one bad file cannot fail the rest of a session.+ */+ private readonly rejectedAttachments = new RejectedAttachments();private capabilityCache: { gatewayUrl: string; accountKey: string; version: 2 | 3; expiresAt: number } | null = null;private capabilityProbe: { key: string; promise: Promise<2 | 3> } | null = null;@@ -1021,7 +1032,9 @@ export class OmniRushGatewayBroker {// A picked sub-agent model the gateway refused moments ago is not tried// again for every step of a running sub-agent: it goes to the main model.const skipped = requestBody === undefined ? null : this.skipRefusedSubagentModel(request, normalizedPath, requestBody);- const body = skipped?.body ?? requestBody;+ let body = skipped?.body ?? requestBody;+ const attachments = this.leaveOutRefusedAttachments(normalizedPath, body);+ if (attachments) body = attachments.body;let spent = this.state.accessToken;let response = await this.forwardModelRequest(request, normalizedPath, body, spent);// Two rounds at most: the first may only adopt a pair the desktop rotated,@@ -1057,6 +1070,11 @@ export class OmniRushGatewayBroker {},}, { status: 401 });}+ if (attachments) {+ const retried = await this.retryWithoutRefusedAttachments(request, normalizedPath, attachments, response);+ response = retried.response;+ body = retried.body;+ }if (skipped) {this.reportSubagentFallback(request, { ...skipped.event, status: response.status, ok: response.ok });} else if (!response.ok && response.status !== 401 && body !== undefined) {@@ -1082,6 +1100,58 @@ export class OmniRushGatewayBroker {});}+ /**+ * A model request's attachments, with the ones the model refused earlier+ * replaced; null for a request that is not JSON or carries none.+ */+ private leaveOutRefusedAttachments(path: string, body: ArrayBuffer | string | undefined): { parsed: unknown; attachments: Attachment[]; body: ArrayBuffer | string } | null {+ if (body === undefined || (path !== "responses" && path !== "responses/compact")) return null;+ let parsed: unknown;+ try {+ parsed = JSON.parse(typeof body === "string" ? body : new TextDecoder().decode(body));+ } catch {+ return null;+ }+ if (!parsed || typeof parsed !== "object") return null;+ const prepared = this.rejectedAttachments.prepare(parsed);+ if (prepared.attachments.length === 0) return null;+ if (prepared.replaced.length > 0) {+ this.log?.("info", "omnirush.ai: left out attachments the model refused earlier", { count: prepared.replaced.length });+ return { parsed, attachments: prepared.attachments, body: JSON.stringify(prepared.body) };+ }+ return { parsed, attachments: prepared.attachments, body };+ }++ /**+ * The model refused an attachment (400 "The file you uploaded is badly+ * formatted or corrupted"): the suspects are left out and the request sent+ * again, so the turn goes on with a "[file could not be read: x]" note. An+ * answer the model accepted marks its attachments as good.+ */+ private async retryWithoutRefusedAttachments(+ request: Request,+ path: string,+ state: { parsed: unknown; attachments: Attachment[]; body: ArrayBuffer | string },+ first: Response,+ ): Promise<{ response: Response; body: ArrayBuffer | string }> {+ let response = first;+ let { attachments, body } = state;+ for (let round = 0; response.status === 400 && round < MAX_ATTACHMENT_RETRIES && this.state; round += 1) {+ const raw = new Uint8Array(await response.arrayBuffer().catch(() => new ArrayBuffer(0)));+ const kept = new Response(raw.byteLength ? raw : null, { status: response.status, statusText: response.statusText, headers: response.headers });+ const message = new TextDecoder().decode(raw.subarray(0, MAX_ATTACHMENT_ERROR_BYTES));+ const suspects = isRejectedAttachmentError(response.status, message) ? this.rejectedAttachments.reject(attachments, message) : [];+ if (suspects.length === 0) return { response: kept, body };+ const prepared = this.rejectedAttachments.prepare(state.parsed);+ attachments = prepared.attachments;+ body = JSON.stringify(prepared.body);+ this.log?.("warn", "omnirush.ai: the model refused an attachment; sending the request again without it", { path, left_out: suspects.length });+ response = await this.forwardModelRequest(request, path, body, this.state.accessToken);+ }+ if (response.ok) this.rejectedAttachments.accept(attachments.filter((item) => !this.rejectedAttachments.isBad(item.fingerprint)));+ return { response, body };+ }+/*** One session uploader envelope. The deadline grows with the envelope's size* (session-upload-budget.ts), and the retry with a refreshed bearer gets adiff --git a/apps/server/src/rejected-attachments.test.ts b/apps/server/src/rejected-attachments.test.tsnew file mode 100644index 00000000..e935e1f9--- /dev/null+++ b/apps/server/src/rejected-attachments.test.ts@@ -0,0 +1,106 @@+import { describe, expect, test } from "bun:test";++import { OmniRushGatewayBroker } from "./omnirush-gateway-broker.js";+import { RejectedAttachments, attachmentsIn, isRejectedAttachmentError, replaceAttachments } from "./rejected-attachments.js";++// The 400 a tester got after the read tool returned an invalid PDF (every later turn failed with it).+const CORRUPT = "The file you uploaded is badly formatted or corrupted. Please fix the file and try again.";+const BAD_PDF = "data:application/pdf;base64,JVBERi0xLjQKJeLjz9MKMSAwIG9iago8PCAvVHlwZSAvQ2F0YWxvZyA+PgplbmRvYmoKdHJhaWxlcgo8PCAvUm9vdCAxIDAgUiA+PgolJUVPRgo=";+const IMAGE = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwAEhQGAhKmMIQAAAABJRU5ErkJggg==";++function responsesBody(prompt: string, withPdf = true) {+ return {+ model: "gpt-6-sol",+ stream: true,+ input: [+ { role: "user", content: [{ type: "input_text", text: "Open bad.pdf with your file read tool" }] },+ { type: "function_call", call_id: "call_1", name: "read", arguments: "{\"filePath\":\"bad.pdf\"}" },+ { type: "function_call_output", call_id: "call_1", output: "PDF read successfully" },+ ...(withPdf ? [{ role: "user", content: [{ type: "input_file", filename: "bad.pdf", file_data: BAD_PDF }, { type: "input_image", image_url: IMAGE }] }] : []),+ { role: "user", content: [{ type: "input_text", text: prompt }] },+ ],+ };+}++describe("rejected attachments", () => {+ test("a 400 about a file or image is recognised; other errors are not", () => {+ expect(isRejectedAttachmentError(400, CORRUPT)).toBe(true);+ expect(isRejectedAttachmentError(400, "Invalid image data")).toBe(true);+ expect(isRejectedAttachmentError(400, "Unsupported parameter: 'max_output_tokens'")).toBe(false);+ expect(isRejectedAttachmentError(500, CORRUPT)).toBe(false);+ });++ test("a new PDF and a new image refused for \"the file\": only the PDF is suspected", () => {+ const both = attachmentsIn(responsesBody("x"));+ expect(new RejectedAttachments().reject(both, CORRUPT).map((item) => item.name)).toEqual(["bad.pdf"]);+ expect(new RejectedAttachments().reject(both, "Invalid image data").map((item) => item.name)).toEqual(["image"]);+ expect(new RejectedAttachments().reject(both, "Invalid content").map((item) => item.name)).toEqual(["bad.pdf", "image"]);+ });++ test("the suspects are the attachments no accepted request carried; they are replaced from then on", () => {+ const guard = new RejectedAttachments();+ guard.accept(attachmentsIn({ input: [{ type: "input_image", image_url: IMAGE }] }));+ const prepared = guard.prepare(responsesBody("two"));+ expect(guard.reject(prepared.attachments).map((item) => item.name)).toEqual(["bad.pdf"]);+ const next = guard.prepare(responsesBody("three"));+ expect(next.replaced).toEqual(["bad.pdf"]);+ expect(JSON.stringify(next.body)).toContain("[file could not be read: bad.pdf]");+ expect(JSON.stringify(next.body)).toContain(IMAGE);+ expect(replaceAttachments(next.body, () => false).body).toBe(next.body);+ });++ test("the broker sends the refused request again without the file, and leaves it out of later turns", async () => {+ const sent: string[] = [];+ const broker = new OmniRushGatewayBroker({+ credentials: { gatewayUrl: "https://gateway.example/omnirush/v1", accessToken: "access-token", refreshToken: "refresh-token" },+ engineToken: "local-engine-token",+ fetch: async (_input, init) => {+ const raw = init?.body;+ const text = typeof raw === "string" ? raw : new TextDecoder().decode(raw as ArrayBuffer);+ sent.push(text);+ if (text.includes("data:application/pdf")) {+ return new Response(JSON.stringify({ error: { message: CORRUPT, type: "invalid_request_error" } }), { status: 400, headers: { "content-type": "text/event-stream" } });+ }+ return new Response("data: ok\n\n", { headers: { "content-type": "text/event-stream" } });+ },+ });+ const turn = (prompt: string) => broker.handle(new Request("http://127.0.0.1/omnirush-gateway/v1/responses", {+ method: "POST",+ headers: { Authorization: "Bearer local-engine-token", "Content-Type": "application/json" },+ body: JSON.stringify(responsesBody(prompt)),+ }), "responses");++ const first = await turn("what did you get?");+ expect(first.status).toBe(200);+ expect(await first.text()).toContain("data: ok");+ expect(sent.length).toBe(2);+ expect(sent[1]).toContain("[file could not be read: bad.pdf]");+ expect(sent[1]).not.toContain("data:application/pdf");+ expect(sent[1]).toContain(IMAGE);++ const second = await turn("Create ok.txt containing ok");+ expect(second.status).toBe(200);+ await second.text();+ expect(sent.length).toBe(3);+ expect(sent[2]).not.toContain("data:application/pdf");+ });++ test("a 400 that is not about an attachment passes through once", async () => {+ let calls = 0;+ const broker = new OmniRushGatewayBroker({+ credentials: { gatewayUrl: "https://gateway.example/omnirush/v1", accessToken: "access-token", refreshToken: "refresh-token" },+ engineToken: "local-engine-token",+ fetch: async () => {+ calls += 1;+ return Response.json({ error: { message: "Unsupported parameter: 'max_output_tokens'" } }, { status: 400 });+ },+ });+ const response = await broker.handle(new Request("http://127.0.0.1/omnirush-gateway/v1/responses", {+ method: "POST",+ headers: { Authorization: "Bearer local-engine-token", "Content-Type": "application/json" },+ body: JSON.stringify(responsesBody("x")),+ }), "responses");+ expect(response.status).toBe(400);+ expect(calls).toBe(1);+ });+});diff --git a/apps/server/src/rejected-attachments.ts b/apps/server/src/rejected-attachments.tsnew file mode 100644index 00000000..1eb26cf2--- /dev/null+++ b/apps/server/src/rejected-attachments.ts@@ -0,0 +1,230 @@+// An attachment the model API refuses (a corrupt PDF the read tool returned,+// an image that is not one) stays in the conversation, so every later request+// carries it again and fails with the same 400: one bad file ends the session.+//+// RejectedAttachments remembers the attachments the model refused (by the+// SHA-256 of their data) and replaces them, in every later request, with a+// short text part: `[file could not be read: <name>]`. Which part was refused+// is not named by the API, so the suspects of a refusal are the request's+// attachments that no earlier accepted request carried (all of them when every+// one was accepted before). The gateway broker (omnirush-gateway-broker.ts)+// retries the failing request at once without the part. Kept in step with the+// CLI's copy (omnirush-cli assets/extensions/omnirush/rejected-attachments.js).+//+// Request shapes understood (anywhere in the JSON body):+// Responses API { type: "input_file", filename, file_data | file_url | file_id }+// { type: "input_image", image_url | file_id }+// Chat Completions { type: "image_url", image_url: { url } }+// { type: "file", file: { filename, file_data | file_id } }+// Messages / pi { type: "image" | "document", source: { data | url } }+// { type: "image", data, mimeType }++import { createHash } from "node:crypto";++const MAX_WALK_DEPTH = 64;++/**+ * Whether a model API error says an attachment could not be used: a 400+ * whose message names a file/image/PDF and calls it invalid, corrupt,+ * unreadable or unsupported.+ */+export function isRejectedAttachmentError(status: unknown, message: unknown): boolean {+ if (Number(status) !== 400) return false;+ const text = String(message ?? "").slice(0, 8192);+ if (!/\b(?:file|files|image|images|pdf|attachment|document|media|input_file|input_image|image_url|file_data)\b/i.test(text)) return false;+ return /invalid|corrupt|badly formatted|malformed|could not (?:be )?(?:process|read|decod|pars|open|load)|unable to (?:process|read|decod|pars|open|load)|failed to (?:process|read|decod|pars|open|load)|not a valid|unsupported|is not supported|cannot be (?:processed|read|decoded|parsed)|did not (?:contain|look like)|no pages/i.test(text);+}++function isRecord(value: unknown): value is Record<string, any> {+ return typeof value === "object" && value !== null && !Array.isArray(value);+}++function dataUrlName(url: unknown): string | null {+ const mime = /^data:([^;,]+)/i.exec(String(url))?.[1];+ if (!mime) return null;+ return mime.startsWith("image/") ? "image" : mime === "application/pdf" ? "document.pdf" : "attachment";+}++export type AttachmentStyle = "input_text" | "text";+export type AttachmentKind = "image" | "file";+export type Attachment = { fingerprint: string; name: string; kind: AttachmentKind };++function kindOf(data: string): AttachmentKind {+ return /^data:image\//i.test(data) ? "image" : "file";+}++/**+ * Which kind of attachment an error message blames: "image" when it names an+ * image and no file/PDF/document, "file" for the reverse, else null (either).+ */+export function blamedKind(message: unknown): AttachmentKind | null {+ const text = String(message ?? "");+ const image = /\bimages?\b|image_url|input_image/i.test(text);+ const file = /\b(?:files?|pdfs?|documents?)\b|input_file|file_data/i.test(text);+ if (image && !file) return "image";+ if (file && !image) return "file";+ return null;+}++/** The attachment a content part carries ({ data, name, style, kind }), or null. */+export function attachmentOf(part: unknown): { data: string; name: string; style: AttachmentStyle; kind: AttachmentKind } | null {+ if (!isRecord(part) || typeof part.type !== "string") return null;+ const str = (value: unknown): string | null => (typeof value === "string" && value.length > 0 ? value : null);+ switch (part.type) {+ case "input_file": {+ const data = str(part.file_data) ?? str(part.file_url) ?? str(part.file_id);+ return data ? { data, name: str(part.filename) ?? dataUrlName(data) ?? "file", style: "input_text", kind: kindOf(data) } : null;+ }+ case "input_image": {+ const url = isRecord(part.image_url) ? part.image_url.url : part.image_url;+ const data = str(url) ?? str(part.file_id);+ return data ? { data, name: str(part.filename) ?? "image", style: "input_text", kind: "image" } : null;+ }+ case "image_url": {+ const url = isRecord(part.image_url) ? part.image_url.url : part.image_url;+ const data = str(url);+ return data ? { data, name: "image", style: "text", kind: "image" } : null;+ }+ case "file": {+ const file = isRecord(part.file) ? part.file : null;+ const data = file ? str(file.file_data) ?? str(file.file_id) ?? str(file.url) : null;+ return data && file ? { data, name: str(file.filename) ?? dataUrlName(data) ?? "file", style: "text", kind: kindOf(data) } : null;+ }+ case "image":+ case "document": {+ const source = isRecord(part.source) ? part.source : null;+ const data = str(part.data) ?? (source ? str(source.data) ?? str(source.url) ?? str(source.file_id) : null);+ const name = str(part.filename) ?? str(part.name) ?? str(part.title) ?? (part.type === "image" ? "image" : "document");+ return data ? { data, name, style: "text", kind: part.type === "image" ? "image" : kindOf(data) } : null;+ }+ default:+ return null;+ }+}++export function fingerprint(data: string): string {+ return createHash("sha256").update(String(data)).digest("hex");+}++/** The placeholder that stands in for a refused attachment. */+export function placeholderText(name: string): string {+ const clean = String(name || "file").replace(/[\r\n\]]+/g, " ").slice(0, 200);+ return `[file could not be read: ${clean}]`;+}++/** Every attachment of a request body (or message list), in order: { fingerprint, name }. */+export function attachmentsIn(body: unknown): Attachment[] {+ const out: Attachment[] = [];+ const walk = (value: unknown, depth: number): void => {+ if (depth > MAX_WALK_DEPTH) return;+ if (Array.isArray(value)) {+ for (const item of value) walk(item, depth + 1);+ return;+ }+ if (!isRecord(value)) return;+ const attachment = attachmentOf(value);+ if (attachment) {+ out.push({ fingerprint: fingerprint(attachment.data), name: attachment.name, kind: attachment.kind });+ return;+ }+ for (const key of Object.keys(value)) walk(value[key], depth + 1);+ };+ walk(body, 0);+ return out;+}++/**+ * A copy of `body` with every attachment whose fingerprint `isBad` accepts+ * replaced by a text part of the same API style; `replaced` lists their names.+ * The body is returned as is (same object) when nothing is replaced.+ */+export function replaceAttachments<T>(body: T, isBad: (fingerprint: string) => boolean): { body: T; replaced: string[] } {+ const replaced: string[] = [];+ const walk = (value: unknown, depth: number): unknown => {+ if (depth > MAX_WALK_DEPTH) return value;+ if (Array.isArray(value)) {+ let changed = false;+ const next = value.map((item) => {+ const out = walk(item, depth + 1);+ if (out !== item) changed = true;+ return out;+ });+ return changed ? next : value;+ }+ if (!isRecord(value)) return value;+ const attachment = attachmentOf(value);+ if (attachment) {+ if (!isBad(fingerprint(attachment.data))) return value;+ replaced.push(attachment.name);+ return { type: attachment.style, text: placeholderText(attachment.name) };+ }+ let next: Record<string, unknown> = value;+ for (const key of Object.keys(value)) {+ const out = walk(value[key], depth + 1);+ if (out !== value[key]) {+ if (next === value) next = { ...value };+ next[key] = out;+ }+ }+ return next;+ };+ const out = walk(body, 0) as T;+ return { body: out, replaced };+}++export class RejectedAttachments {+ /** Carried by a request the model accepted. */+ private readonly accepted = new Set<string>();+ /** fingerprint -> name: refused; replaced from now on. */+ private readonly bad = new Map<string, string>();++ constructor(private readonly maxEntries = 10_000) {}++ get size(): number {+ return this.bad.size;+ }++ isBad(fp: string): boolean {+ return this.bad.has(fp);+ }++ /** The request's attachments (fingerprints) and its body with the refused ones replaced. */+ prepare<T>(body: T): { body: T; attachments: Attachment[]; replaced: string[] } {+ const attachments = attachmentsIn(body);+ if (attachments.length === 0 || this.bad.size === 0) return { body, attachments, replaced: [] };+ const { body: next, replaced } = replaceAttachments(body, (fp) => this.bad.has(fp));+ return { body: next, attachments, replaced };+ }++ /** The model answered a request that carried these attachments. */+ accept(attachments: readonly Attachment[]): void {+ for (const { fingerprint: fp } of attachments) {+ if (this.bad.has(fp)) continue;+ if (this.accepted.size >= this.maxEntries) this.accepted.clear();+ this.accepted.add(fp);+ }+ }++ /**+ * The model refused a request carrying these attachments for one of them:+ * marks the suspects as refused and returns them ([] when there is nothing+ * left to leave out). Suspects: the attachments not refused yet, narrowed to+ * the ones no accepted request carried, then to the kind the error message+ * blames (`message`: "The file you uploaded..." blames files, not images),+ * each step only when it leaves some.+ */+ reject(attachments: readonly Attachment[], message = ""): Attachment[] {+ const open = attachments.filter((item) => !this.bad.has(item.fingerprint));+ const fresh = open.filter((item) => !this.accepted.has(item.fingerprint));+ let suspects = fresh.length > 0 ? fresh : open;+ const blamed = blamedKind(message);+ const ofKind = blamed ? suspects.filter((item) => item.kind === blamed) : [];+ if (ofKind.length > 0) suspects = ofKind;+ for (const item of suspects) {+ if (this.bad.size >= this.maxEntries) break;+ this.bad.set(item.fingerprint, item.name);+ this.accepted.delete(item.fingerprint);+ }+ return suspects;+ }+}