A desktop coding agent with free access to frontier models.
TypeScript
fix(session-uploader): keep binaries out of text, give skipped turn-diff entries a reason
ogsamrat6fd8dce089b4
diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.tsindex 88c97b74..355c8f53 100644--- a/apps/server/src/session-uploader.ts+++ b/apps/server/src/session-uploader.ts@@ -6,6 +6,7 @@ import { release as osRelease, tmpdir } from "node:os";import nodePath, { basename, dirname, extname, join, relative, resolve, sep, type PlatformPath } from "node:path";import { promisify } from "node:util";import { createZstdCompress } from "node:zlib";+import { isUtf8 } from "node:buffer";import { minimatch } from "minimatch";import { SESSION_UPLOAD_BUDGET, SESSION_UPLOAD_ENDPOINT_PATH, sessionUploadTimeoutMs, type SessionUploadBudget } from "./session-upload-budget.js";@@ -1488,7 +1489,14 @@ export function stripRemoteUserinfo(url: string): string {return trimmed;}-function isBinary(buffer: Buffer): boolean {+/**+ * Whether `buffer` holds binary content: a NUL or many control bytes in its+ * first 8 KiB, or bytes that are not valid UTF-8 anywhere (a PDF, an image).+ * Text is uploaded as UTF-8, so a file that does not decode strictly would be+ * stored altered (each invalid byte becoming U+FFFD). `partial` is set for a+ * head cut from a longer file, which may end inside a character.+ */+function isBinary(buffer: Buffer, partial = false): boolean {const length = Math.min(buffer.length, 8_192);let suspicious = 0;// An indexed loop: iterating a Buffer is several times slower, once per file.@@ -1497,7 +1505,21 @@ function isBinary(buffer: Buffer): boolean {if (byte === 0) return true;if (byte < 7 || (byte > 13 && byte < 32)) suspicious += 1;}- return length > 0 && suspicious / length > 0.1;+ if (length > 0 && suspicious / length > 0.1) return true;+ return !isValidUtf8(buffer, partial);+}++/** Strict UTF-8 validation (native); with `partial`, an incomplete last character is allowed. */+function isValidUtf8(buffer: Buffer, partial: boolean): boolean {+ if (isUtf8(buffer)) return true;+ if (!partial) return false;+ for (let back = 1; back <= 3 && back <= buffer.length; back += 1) {+ const byte = buffer[buffer.length - back]!;+ if ((byte & 0xc0) === 0x80) continue;+ const length = byte >= 0xf0 ? 4 : byte >= 0xe0 ? 3 : byte >= 0xc0 ? 2 : 1;+ return length > back && isUtf8(buffer.subarray(0, buffer.length - back));+ }+ return false;}const PATH_FIELD_NAMES = new Set([@@ -3006,7 +3028,8 @@ function writtenThisTurn(turnStartedAt: number | null, cached: HashCacheEntry |async function startsBinary(absolute: string): Promise<boolean> {const head = Buffer.alloc(8_192);try {- return isBinary(head.subarray(0, await readInto(absolute, head)));+ const read = await readInto(absolute, head);+ return isBinary(head.subarray(0, read), read === head.length);} catch {return false;}@@ -4537,8 +4560,9 @@ export class SessionUploader {return base === null ? { ...input, status: "no_base" } : { ...input, status: "deleted", before: base };}if (!file.isFile() || file.isSymbolicLink()) return null;- if (file.size <= MAX_UPLOAD_FILE_BYTES && !state.cache.get(path)?.binary && !(await startsBinary(absolute))) return null;- return { ...input, after_sha256: null, status: "skipped" };+ if (file.size > MAX_UPLOAD_FILE_BYTES) return { ...input, after_sha256: null, status: "skipped", reason: "too_large" };+ if (!state.cache.get(path)?.binary && !(await startsBinary(absolute))) return null;+ return { ...input, after_sha256: null, status: "skipped", reason: "binary" };}/** The text a manifest entry describes when the base store no longer holds it: read again, if the file still hashes to it. */@@ -4818,7 +4842,17 @@ export class SessionUploader {const written = writtenThisTurn(state.turnStartedAt, state.cache.get(path), file);if (written && state.turnStartedAt !== null) state.turnWrittenPaths.set(path, Date.now());if (file.isFile() && !file.isSymbolicLink() && file.size > MAX_CHANGE_JOURNAL_BYTES && written) state.turnSkipped.set(path, file.mtimeMs);- if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_UPLOAD_FILE_BYTES) {+ if (!file.isFile() || file.isSymbolicLink()) {+ // A directory or a symlink is not a workspace file (the snapshot lists+ // neither): nothing to journal, and a file it replaced leaves the manifest.+ const previous = state.changeJournal.get(path);+ if (previous) {+ state.changeJournalBytes -= Buffer.byteLength(JSON.stringify(previous));+ state.changeJournal.delete(path);+ }+ return;+ }+ if (file.size > MAX_UPLOAD_FILE_BYTES) {entry = { path, at: new Date().toISOString(), status: "skipped" };} else if (file.size > MAX_CHANGE_JOURNAL_BYTES) {// Larger than the whole journal: it could only evict every other entrydiff --git a/apps/server/src/turn-diff.test.ts b/apps/server/src/turn-diff.test.tsindex faeab0a7..ada88fa2 100644--- a/apps/server/src/turn-diff.test.ts+++ b/apps/server/src/turn-diff.test.ts@@ -1,6 +1,6 @@import { afterEach, describe, expect, test } from "bun:test";import { createHash } from "node:crypto";-import { mkdtemp, readdir, rm, stat, unlink, utimes, writeFile } from "node:fs/promises";+import { mkdir, mkdtemp, readdir, rm, stat, symlink, unlink, utimes, writeFile } from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";import { zstdDecompressSync } from "node:zlib";@@ -250,6 +250,7 @@ describe("turn.diff events", () => {deletions: 1,diff: "--- a/app.ts\n+++ b/app.ts\n@@ -1,2 +1,2 @@\n-export const answer = 41;\n+export const answer = 42;\n export const name = \"omnirush\";\n",truncated: false,+ reason: null,},{path: "new.md",@@ -260,6 +261,7 @@ describe("turn.diff events", () => {deletions: 0,diff: "--- /dev/null\n+++ b/new.md\n@@ -0,0 +1,2 @@\n+fresh\n+notes\n",truncated: false,+ reason: null,},{path: "old.md",@@ -270,6 +272,7 @@ describe("turn.diff events", () => {deletions: 1,diff: "--- a/old.md\n+++ /dev/null\n@@ -1 +0,0 @@\n-retired\n",truncated: false,+ reason: null,},]);});@@ -294,10 +297,45 @@ describe("turn.diff events", () => {await writeFile(join(root, "chart.png"), Buffer.from([0x89, 0x50, 0x4e, 0x47, 0, 0, 0, 13, 0xff]));});expect(event.files).toEqual([- { path: "chart.png", status: "skipped", before_sha256: null, after_sha256: null, additions: null, deletions: null, diff: null, truncated: false },+ { path: "chart.png", status: "skipped", before_sha256: null, after_sha256: null, additions: null, deletions: null, diff: null, truncated: false, reason: "binary" },]);});+ test("marks a PDF (invalid UTF-8, few control bytes) and an oversized file the turn wrote as skipped, with their reason", async () => {+ const root = await tempDir("omnirush-turn-diff-kinds-");+ await writeFile(join(root, "readme.txt"), "hello\n");+ const pdf = Buffer.concat([Buffer.from("%PDF-1.4\n%"), Buffer.from([0xe2, 0xe3, 0xcf, 0xd3]), Buffer.from("\n1 0 obj\n<< /Type /Catalog >>\nendobj\n%%EOF\n")]);+ const { sessionUploader, uploads } = recordingUploader();+ const sessionId = "session-turn-diff-kinds";+ sessionUploader.startSession(sessionId, "workspace-turn-diff", root);+ sessionUploader.captureSnapshot(sessionId, "prompt");+ await sessionUploader.idle(sessionId);+ await writeFile(join(root, "sample.pdf"), pdf);+ await writeFile(join(root, "dump.log"), "x".repeat(5 * 1024 * 1024));+ // Neither a directory nor a symlink is a workspace file: no entry anywhere.+ await mkdir(join(root, "made"));+ await symlink(join(root, "readme.txt"), join(root, "link.txt"));+ await delay(300);+ sessionUploader.captureSnapshot(sessionId, "turn_completed");+ sessionUploader.flushTrace(sessionId, { messages: [] });+ await sessionUploader.idle(sessionId);+ await sessionUploader.stop();+ const [event] = turnDiffEvents(uploads);+ expect(event!.files.map((file) => [file.path, file.status, file.reason, file.diff])).toEqual([+ ["dump.log", "skipped", "too_large", null],+ ["sample.pdf", "skipped", "binary", null],+ ]);+ const envelopes = uploads as unknown as Array<{ snapshot_type: string; files: Array<{ path: string; content: string }> }>;+ // The PDF's bytes never travel as (altered) text.+ expect(envelopes.flatMap((envelope) => envelope.files).some((file) => file.path === "sample.pdf")).toBe(false);+ const journal = envelopes+ .filter((envelope) => envelope.snapshot_type === "change" || envelope.snapshot_type === "end")+ .flatMap((envelope) => envelope.files.filter((file) => file.path === "__omnirush__/changes.json"))+ .flatMap((file) => (JSON.parse(file.content) as { entries: Array<{ path: string; status: string }> }).entries);+ expect([...new Set(journal.filter((entry) => entry.status === "skipped").map((entry) => entry.path))].sort()).toEqual(["dump.log", "sample.pdf"]);+ expect(journal.some((entry) => entry.path === "made" || entry.path === "link.txt")).toBe(false);+ });+test("keeps a redacted secret redacted on both sides of the diff", async () => {const root = await tempDir("omnirush-turn-diff-secret-");await writeFile(join(root, "config.txt"), "OPENAI_API_KEY=sk-1234567890abcdefghijklmnop\nmode=dev\n");@@ -324,9 +362,9 @@ describe("turn.diff events", () => {await unlink(join(root, "gone.txt"));}, { snapshotMaxBytes: NO_CONTENT });expect(event.files).toEqual([- { path: "added.txt", status: "added", before_sha256: null, after_sha256: sha256("brand new\n"), additions: 1, deletions: 0, diff: "--- /dev/null\n+++ b/added.txt\n@@ -0,0 +1 @@\n+brand new\n", truncated: false },- { path: "big.txt", status: "no_base", before_sha256: sha256("before\n"), after_sha256: sha256("after\n"), additions: null, deletions: null, diff: null, truncated: false },- { path: "gone.txt", status: "no_base", before_sha256: sha256("gone\n"), after_sha256: null, additions: null, deletions: null, diff: null, truncated: false },+ { path: "added.txt", status: "added", before_sha256: null, after_sha256: sha256("brand new\n"), additions: 1, deletions: 0, diff: "--- /dev/null\n+++ b/added.txt\n@@ -0,0 +1 @@\n+brand new\n", truncated: false, reason: null },+ { path: "big.txt", status: "no_base", before_sha256: sha256("before\n"), after_sha256: sha256("after\n"), additions: null, deletions: null, diff: null, truncated: false, reason: null },+ { path: "gone.txt", status: "no_base", before_sha256: sha256("gone\n"), after_sha256: null, additions: null, deletions: null, diff: null, truncated: false, reason: null },]);});@@ -339,7 +377,7 @@ describe("turn.diff events", () => {await utimes(join(root, "early.txt"), later, later);const event = await turnDiff(root, async () => undefined);expect(event.files).toEqual([- { path: "early.txt", status: "no_base", before_sha256: null, after_sha256: sha256("written by the turn\n"), additions: null, deletions: null, diff: null, truncated: false },+ { path: "early.txt", status: "no_base", before_sha256: null, after_sha256: sha256("written by the turn\n"), additions: null, deletions: null, diff: null, truncated: false, reason: null },]);});@@ -381,6 +419,7 @@ describe("turn.diff events", () => {deletions: 1,diff: "--- a/app.txt\n+++ b/app.txt\n@@ -1 +1 @@\n-before\n+after\n",truncated: false,+ reason: null,}],file_count: 1,omitted_file_count: 0,@@ -461,12 +500,13 @@ describe("turn.diff events", () => {await writeFile(join(root, "huge.txt"), Array.from({ length: 40_000 }, (_, index) => `generated line ${index}`).join("\n"));});const [file] = event.files;- expect(file).toMatchObject({ path: "huge.txt", status: "modified", additions: 40_000, deletions: 0, truncated: true });+ expect(file).toMatchObject({ path: "huge.txt", status: "modified", additions: 40_000, deletions: 0, truncated: true, reason: "too_large" });expect(Buffer.byteLength(JSON.stringify(file!.diff))).toBeLessThanOrEqual(MAX_TURN_DIFF_FILE_BYTES + 2);expect(file!.diff!.endsWith("\n")).toBe(true);expect(event.truncated).toBe(true);- const builder = new TurnDiffBuilder(4 * 1024, 1024);+ // The cap leaves room for each entry's reason.+ const builder = new TurnDiffBuilder(4 * 1024 + 128, 1024);for (let index = 0; index < 10; index += 1) {builder.add({ path: `f${index}.txt`, status: "added", before_sha256: null, after_sha256: null, before: null, after: "line\n".repeat(1_000) });}@@ -478,7 +518,8 @@ describe("turn.diff events", () => {expect(capped).toMatchObject({ file_count: 12, omitted_file_count: 7 });expect(capped.files.map((entry) => entry.path)).toEqual(["f0.txt", "f1.txt", "f2.txt", "logo.png", "evicted.txt"]);expect(capped.files.slice(0, 3).every((entry) => entry.truncated)).toBe(true);- expect(traceEventBytes(capped)).toBeLessThanOrEqual(4 * 1024);+ expect(capped.files.map((entry) => entry.reason)).toEqual(["too_large", "too_large", "too_large", "binary", null]);+ expect(traceEventBytes(capped)).toBeLessThanOrEqual(4 * 1024 + 128);expect(capped.truncated).toBe(true);});diff --git a/apps/server/src/turn-diff.ts b/apps/server/src/turn-diff.tsindex bcd8635e..9093f03d 100644--- a/apps/server/src/turn-diff.ts+++ b/apps/server/src/turn-diff.ts@@ -50,6 +50,14 @@ const SHA256_NAME = /^[0-9a-f]{64}$/;export type TurnDiffStatus = "added" | "modified" | "deleted" | "skipped" | "no_base";+/**+ * Why an entry carries no diff, or only part of one: a skipped file is+ * "binary" or "too_large" (over the session uploader's per-file cap); a cut+ * diff is "too_large" (past MAX_TURN_DIFF_FILE_BYTES) or "record_cap" (past+ * what the event's budget had left).+ */+export type TurnDiffReason = "binary" | "too_large" | "record_cap";+/** One file of a "turn.diff" event. */export type TurnDiffFile = {/** Workspace-relative, redacted as upload paths are. */@@ -69,6 +77,8 @@ export type TurnDiffFile = {diff: string | null;/** The diff was cut at MAX_TURN_DIFF_FILE_BYTES or at the event's remaining budget. */truncated: boolean;+ /** Why the file was skipped or its diff cut; null otherwise. */+ reason: TurnDiffReason | null;};export type TurnDiffEvent = {@@ -94,6 +104,8 @@ export type TurnDiffInput = {after_sha256: string | null;before: string | null;after: string | null;+ /** A skipped file: binary, or over the per-file cap. */+ reason?: TurnDiffReason;};type Lines = { lines: string[]; eofNewline: boolean };@@ -384,6 +396,8 @@ export class TurnDiffBuilder {deletions: hasText ? before?.length ?? 0 : null,diff: null,truncated: false,+ // At its widest while the entry is measured.+ reason: input.status === "skipped" ? input.reason ?? "binary" : hasText ? "record_cap" : null,};// The entry without its diff, plus the comma between entries.const overhead = Buffer.byteLength(JSON.stringify(file)) + 1;@@ -391,12 +405,14 @@ export class TurnDiffBuilder {if (room < (hasText ? MIN_DIFF_ROOM_BYTES : 0)) return this.omit();if (hasText) {const started = performance.now();- const result = unifiedDiff(input.path, before, after, Math.min(this.maxFileBytes, room));+ const limit = Math.min(this.maxFileBytes, room);+ const result = unifiedDiff(input.path, before, after, limit);this.diffMs += performance.now() - started;file.additions = result.additions;file.deletions = result.deletions;file.diff = result.diff;file.truncated = result.truncated;+ file.reason = result.truncated ? (limit < this.maxFileBytes ? "record_cap" : "too_large") : null;if (result.truncated) this.truncated = true;}this.files.push(file);