Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix(session-uploader): keep binaries out of text, give skipped turn-diff entries a reason

ogsamrat6fd8dce089b4
diff --git a/apps/server/src/session-uploader.ts b/apps/server/src/session-uploader.ts
index 88c97b74..355c8f53 100644
--- a/apps/server/src/session-uploader.ts
+++ b/apps/server/src/session-uploader.ts
@@ -6,6 +6,7 @@ import { release as osRelease, tmpdir } from "node:os";
import nodePath, { basename, dirname, extname, join, relative, resolve, sep, type PlatformPath } from "node:path";
import { promisify } from "node:util";
import { createZstdCompress } from "node:zlib";
+import { isUtf8 } from "node:buffer";
import { minimatch } from "minimatch";
import { SESSION_UPLOAD_BUDGET, SESSION_UPLOAD_ENDPOINT_PATH, sessionUploadTimeoutMs, type SessionUploadBudget } from "./session-upload-budget.js";
@@ -1488,7 +1489,14 @@ export function stripRemoteUserinfo(url: string): string {
return trimmed;
}
-function isBinary(buffer: Buffer): boolean {
+/**
+ * Whether `buffer` holds binary content: a NUL or many control bytes in its
+ * first 8 KiB, or bytes that are not valid UTF-8 anywhere (a PDF, an image).
+ * Text is uploaded as UTF-8, so a file that does not decode strictly would be
+ * stored altered (each invalid byte becoming U+FFFD). `partial` is set for a
+ * head cut from a longer file, which may end inside a character.
+ */
+function isBinary(buffer: Buffer, partial = false): boolean {
const length = Math.min(buffer.length, 8_192);
let suspicious = 0;
// An indexed loop: iterating a Buffer is several times slower, once per file.
@@ -1497,7 +1505,21 @@ function isBinary(buffer: Buffer): boolean {
if (byte === 0) return true;
if (byte < 7 || (byte > 13 && byte < 32)) suspicious += 1;
}
- return length > 0 && suspicious / length > 0.1;
+ if (length > 0 && suspicious / length > 0.1) return true;
+ return !isValidUtf8(buffer, partial);
+}
+
+/** Strict UTF-8 validation (native); with `partial`, an incomplete last character is allowed. */
+function isValidUtf8(buffer: Buffer, partial: boolean): boolean {
+ if (isUtf8(buffer)) return true;
+ if (!partial) return false;
+ for (let back = 1; back <= 3 && back <= buffer.length; back += 1) {
+ const byte = buffer[buffer.length - back]!;
+ if ((byte & 0xc0) === 0x80) continue;
+ const length = byte >= 0xf0 ? 4 : byte >= 0xe0 ? 3 : byte >= 0xc0 ? 2 : 1;
+ return length > back && isUtf8(buffer.subarray(0, buffer.length - back));
+ }
+ return false;
}
const PATH_FIELD_NAMES = new Set([
@@ -3006,7 +3028,8 @@ function writtenThisTurn(turnStartedAt: number | null, cached: HashCacheEntry |
async function startsBinary(absolute: string): Promise<boolean> {
const head = Buffer.alloc(8_192);
try {
- return isBinary(head.subarray(0, await readInto(absolute, head)));
+ const read = await readInto(absolute, head);
+ return isBinary(head.subarray(0, read), read === head.length);
} catch {
return false;
}
@@ -4537,8 +4560,9 @@ export class SessionUploader {
return base === null ? { ...input, status: "no_base" } : { ...input, status: "deleted", before: base };
}
if (!file.isFile() || file.isSymbolicLink()) return null;
- if (file.size <= MAX_UPLOAD_FILE_BYTES && !state.cache.get(path)?.binary && !(await startsBinary(absolute))) return null;
- return { ...input, after_sha256: null, status: "skipped" };
+ if (file.size > MAX_UPLOAD_FILE_BYTES) return { ...input, after_sha256: null, status: "skipped", reason: "too_large" };
+ if (!state.cache.get(path)?.binary && !(await startsBinary(absolute))) return null;
+ return { ...input, after_sha256: null, status: "skipped", reason: "binary" };
}
/** The text a manifest entry describes when the base store no longer holds it: read again, if the file still hashes to it. */
@@ -4818,7 +4842,17 @@ export class SessionUploader {
const written = writtenThisTurn(state.turnStartedAt, state.cache.get(path), file);
if (written && state.turnStartedAt !== null) state.turnWrittenPaths.set(path, Date.now());
if (file.isFile() && !file.isSymbolicLink() && file.size > MAX_CHANGE_JOURNAL_BYTES && written) state.turnSkipped.set(path, file.mtimeMs);
- if (!file.isFile() || file.isSymbolicLink() || file.size > MAX_UPLOAD_FILE_BYTES) {
+ if (!file.isFile() || file.isSymbolicLink()) {
+ // A directory or a symlink is not a workspace file (the snapshot lists
+ // neither): nothing to journal, and a file it replaced leaves the manifest.
+ const previous = state.changeJournal.get(path);
+ if (previous) {
+ state.changeJournalBytes -= Buffer.byteLength(JSON.stringify(previous));
+ state.changeJournal.delete(path);
+ }
+ return;
+ }
+ if (file.size > MAX_UPLOAD_FILE_BYTES) {
entry = { path, at: new Date().toISOString(), status: "skipped" };
} else if (file.size > MAX_CHANGE_JOURNAL_BYTES) {
// Larger than the whole journal: it could only evict every other entry
diff --git a/apps/server/src/turn-diff.test.ts b/apps/server/src/turn-diff.test.ts
index faeab0a7..ada88fa2 100644
--- a/apps/server/src/turn-diff.test.ts
+++ b/apps/server/src/turn-diff.test.ts
@@ -1,6 +1,6 @@
import { afterEach, describe, expect, test } from "bun:test";
import { createHash } from "node:crypto";
-import { mkdtemp, readdir, rm, stat, unlink, utimes, writeFile } from "node:fs/promises";
+import { mkdir, mkdtemp, readdir, rm, stat, symlink, unlink, utimes, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { zstdDecompressSync } from "node:zlib";
@@ -250,6 +250,7 @@ describe("turn.diff events", () => {
deletions: 1,
diff: "--- a/app.ts\n+++ b/app.ts\n@@ -1,2 +1,2 @@\n-export const answer = 41;\n+export const answer = 42;\n export const name = \"omnirush\";\n",
truncated: false,
+ reason: null,
},
{
path: "new.md",
@@ -260,6 +261,7 @@ describe("turn.diff events", () => {
deletions: 0,
diff: "--- /dev/null\n+++ b/new.md\n@@ -0,0 +1,2 @@\n+fresh\n+notes\n",
truncated: false,
+ reason: null,
},
{
path: "old.md",
@@ -270,6 +272,7 @@ describe("turn.diff events", () => {
deletions: 1,
diff: "--- a/old.md\n+++ /dev/null\n@@ -1 +0,0 @@\n-retired\n",
truncated: false,
+ reason: null,
},
]);
});
@@ -294,10 +297,45 @@ describe("turn.diff events", () => {
await writeFile(join(root, "chart.png"), Buffer.from([0x89, 0x50, 0x4e, 0x47, 0, 0, 0, 13, 0xff]));
});
expect(event.files).toEqual([
- { path: "chart.png", status: "skipped", before_sha256: null, after_sha256: null, additions: null, deletions: null, diff: null, truncated: false },
+ { path: "chart.png", status: "skipped", before_sha256: null, after_sha256: null, additions: null, deletions: null, diff: null, truncated: false, reason: "binary" },
]);
});
+ test("marks a PDF (invalid UTF-8, few control bytes) and an oversized file the turn wrote as skipped, with their reason", async () => {
+ const root = await tempDir("omnirush-turn-diff-kinds-");
+ await writeFile(join(root, "readme.txt"), "hello\n");
+ const pdf = Buffer.concat([Buffer.from("%PDF-1.4\n%"), Buffer.from([0xe2, 0xe3, 0xcf, 0xd3]), Buffer.from("\n1 0 obj\n<< /Type /Catalog >>\nendobj\n%%EOF\n")]);
+ const { sessionUploader, uploads } = recordingUploader();
+ const sessionId = "session-turn-diff-kinds";
+ sessionUploader.startSession(sessionId, "workspace-turn-diff", root);
+ sessionUploader.captureSnapshot(sessionId, "prompt");
+ await sessionUploader.idle(sessionId);
+ await writeFile(join(root, "sample.pdf"), pdf);
+ await writeFile(join(root, "dump.log"), "x".repeat(5 * 1024 * 1024));
+ // Neither a directory nor a symlink is a workspace file: no entry anywhere.
+ await mkdir(join(root, "made"));
+ await symlink(join(root, "readme.txt"), join(root, "link.txt"));
+ await delay(300);
+ sessionUploader.captureSnapshot(sessionId, "turn_completed");
+ sessionUploader.flushTrace(sessionId, { messages: [] });
+ await sessionUploader.idle(sessionId);
+ await sessionUploader.stop();
+ const [event] = turnDiffEvents(uploads);
+ expect(event!.files.map((file) => [file.path, file.status, file.reason, file.diff])).toEqual([
+ ["dump.log", "skipped", "too_large", null],
+ ["sample.pdf", "skipped", "binary", null],
+ ]);
+ const envelopes = uploads as unknown as Array<{ snapshot_type: string; files: Array<{ path: string; content: string }> }>;
+ // The PDF's bytes never travel as (altered) text.
+ expect(envelopes.flatMap((envelope) => envelope.files).some((file) => file.path === "sample.pdf")).toBe(false);
+ const journal = envelopes
+ .filter((envelope) => envelope.snapshot_type === "change" || envelope.snapshot_type === "end")
+ .flatMap((envelope) => envelope.files.filter((file) => file.path === "__omnirush__/changes.json"))
+ .flatMap((file) => (JSON.parse(file.content) as { entries: Array<{ path: string; status: string }> }).entries);
+ expect([...new Set(journal.filter((entry) => entry.status === "skipped").map((entry) => entry.path))].sort()).toEqual(["dump.log", "sample.pdf"]);
+ expect(journal.some((entry) => entry.path === "made" || entry.path === "link.txt")).toBe(false);
+ });
+
test("keeps a redacted secret redacted on both sides of the diff", async () => {
const root = await tempDir("omnirush-turn-diff-secret-");
await writeFile(join(root, "config.txt"), "OPENAI_API_KEY=sk-1234567890abcdefghijklmnop\nmode=dev\n");
@@ -324,9 +362,9 @@ describe("turn.diff events", () => {
await unlink(join(root, "gone.txt"));
}, { snapshotMaxBytes: NO_CONTENT });
expect(event.files).toEqual([
- { path: "added.txt", status: "added", before_sha256: null, after_sha256: sha256("brand new\n"), additions: 1, deletions: 0, diff: "--- /dev/null\n+++ b/added.txt\n@@ -0,0 +1 @@\n+brand new\n", truncated: false },
- { path: "big.txt", status: "no_base", before_sha256: sha256("before\n"), after_sha256: sha256("after\n"), additions: null, deletions: null, diff: null, truncated: false },
- { path: "gone.txt", status: "no_base", before_sha256: sha256("gone\n"), after_sha256: null, additions: null, deletions: null, diff: null, truncated: false },
+ { path: "added.txt", status: "added", before_sha256: null, after_sha256: sha256("brand new\n"), additions: 1, deletions: 0, diff: "--- /dev/null\n+++ b/added.txt\n@@ -0,0 +1 @@\n+brand new\n", truncated: false, reason: null },
+ { path: "big.txt", status: "no_base", before_sha256: sha256("before\n"), after_sha256: sha256("after\n"), additions: null, deletions: null, diff: null, truncated: false, reason: null },
+ { path: "gone.txt", status: "no_base", before_sha256: sha256("gone\n"), after_sha256: null, additions: null, deletions: null, diff: null, truncated: false, reason: null },
]);
});
@@ -339,7 +377,7 @@ describe("turn.diff events", () => {
await utimes(join(root, "early.txt"), later, later);
const event = await turnDiff(root, async () => undefined);
expect(event.files).toEqual([
- { path: "early.txt", status: "no_base", before_sha256: null, after_sha256: sha256("written by the turn\n"), additions: null, deletions: null, diff: null, truncated: false },
+ { path: "early.txt", status: "no_base", before_sha256: null, after_sha256: sha256("written by the turn\n"), additions: null, deletions: null, diff: null, truncated: false, reason: null },
]);
});
@@ -381,6 +419,7 @@ describe("turn.diff events", () => {
deletions: 1,
diff: "--- a/app.txt\n+++ b/app.txt\n@@ -1 +1 @@\n-before\n+after\n",
truncated: false,
+ reason: null,
}],
file_count: 1,
omitted_file_count: 0,
@@ -461,12 +500,13 @@ describe("turn.diff events", () => {
await writeFile(join(root, "huge.txt"), Array.from({ length: 40_000 }, (_, index) => `generated line ${index}`).join("\n"));
});
const [file] = event.files;
- expect(file).toMatchObject({ path: "huge.txt", status: "modified", additions: 40_000, deletions: 0, truncated: true });
+ expect(file).toMatchObject({ path: "huge.txt", status: "modified", additions: 40_000, deletions: 0, truncated: true, reason: "too_large" });
expect(Buffer.byteLength(JSON.stringify(file!.diff))).toBeLessThanOrEqual(MAX_TURN_DIFF_FILE_BYTES + 2);
expect(file!.diff!.endsWith("\n")).toBe(true);
expect(event.truncated).toBe(true);
- const builder = new TurnDiffBuilder(4 * 1024, 1024);
+ // The cap leaves room for each entry's reason.
+ const builder = new TurnDiffBuilder(4 * 1024 + 128, 1024);
for (let index = 0; index < 10; index += 1) {
builder.add({ path: `f${index}.txt`, status: "added", before_sha256: null, after_sha256: null, before: null, after: "line\n".repeat(1_000) });
}
@@ -478,7 +518,8 @@ describe("turn.diff events", () => {
expect(capped).toMatchObject({ file_count: 12, omitted_file_count: 7 });
expect(capped.files.map((entry) => entry.path)).toEqual(["f0.txt", "f1.txt", "f2.txt", "logo.png", "evicted.txt"]);
expect(capped.files.slice(0, 3).every((entry) => entry.truncated)).toBe(true);
- expect(traceEventBytes(capped)).toBeLessThanOrEqual(4 * 1024);
+ expect(capped.files.map((entry) => entry.reason)).toEqual(["too_large", "too_large", "too_large", "binary", null]);
+ expect(traceEventBytes(capped)).toBeLessThanOrEqual(4 * 1024 + 128);
expect(capped.truncated).toBe(true);
});
diff --git a/apps/server/src/turn-diff.ts b/apps/server/src/turn-diff.ts
index bcd8635e..9093f03d 100644
--- a/apps/server/src/turn-diff.ts
+++ b/apps/server/src/turn-diff.ts
@@ -50,6 +50,14 @@ const SHA256_NAME = /^[0-9a-f]{64}$/;
export type TurnDiffStatus = "added" | "modified" | "deleted" | "skipped" | "no_base";
+/**
+ * Why an entry carries no diff, or only part of one: a skipped file is
+ * "binary" or "too_large" (over the session uploader's per-file cap); a cut
+ * diff is "too_large" (past MAX_TURN_DIFF_FILE_BYTES) or "record_cap" (past
+ * what the event's budget had left).
+ */
+export type TurnDiffReason = "binary" | "too_large" | "record_cap";
+
/** One file of a "turn.diff" event. */
export type TurnDiffFile = {
/** Workspace-relative, redacted as upload paths are. */
@@ -69,6 +77,8 @@ export type TurnDiffFile = {
diff: string | null;
/** The diff was cut at MAX_TURN_DIFF_FILE_BYTES or at the event's remaining budget. */
truncated: boolean;
+ /** Why the file was skipped or its diff cut; null otherwise. */
+ reason: TurnDiffReason | null;
};
export type TurnDiffEvent = {
@@ -94,6 +104,8 @@ export type TurnDiffInput = {
after_sha256: string | null;
before: string | null;
after: string | null;
+ /** A skipped file: binary, or over the per-file cap. */
+ reason?: TurnDiffReason;
};
type Lines = { lines: string[]; eofNewline: boolean };
@@ -384,6 +396,8 @@ export class TurnDiffBuilder {
deletions: hasText ? before?.length ?? 0 : null,
diff: null,
truncated: false,
+ // At its widest while the entry is measured.
+ reason: input.status === "skipped" ? input.reason ?? "binary" : hasText ? "record_cap" : null,
};
// The entry without its diff, plus the comma between entries.
const overhead = Buffer.byteLength(JSON.stringify(file)) + 1;
@@ -391,12 +405,14 @@ export class TurnDiffBuilder {
if (room < (hasText ? MIN_DIFF_ROOM_BYTES : 0)) return this.omit();
if (hasText) {
const started = performance.now();
- const result = unifiedDiff(input.path, before, after, Math.min(this.maxFileBytes, room));
+ const limit = Math.min(this.maxFileBytes, room);
+ const result = unifiedDiff(input.path, before, after, limit);
this.diffMs += performance.now() - started;
file.additions = result.additions;
file.deletions = result.deletions;
file.diff = result.diff;
file.truncated = result.truncated;
+ file.reason = result.truncated ? (limit < this.maxFileBytes ? "record_cap" : "too_large") : null;
if (result.truncated) this.truncated = true;
}
this.files.push(file);