TypeScript
fix(server): checkpoints no longer rehash every file when one entry is skip-worktree or assume-unchanged (#23624)
Julius Marminge07a1e799d6dd
diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.tsindex 9cb88072194..9262688fd10 100644--- a/apps/server/src/vcs/GitVcsDriver.test.ts+++ b/apps/server/src/vcs/GitVcsDriver.test.ts@@ -597,7 +597,17 @@ it.effect.each(Effect.gen(function* () {const fs = yield* FileSystem.FileSystem;const path = yield* Path.Path;- const driver = yield* GitVcsDriver.makeVcsDriverShape();+ const liveProcess = yield* VcsProcess.VcsProcess;+ let freshReadTrees = 0;+ const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(+ Effect.provideService(VcsProcess.VcsProcess, {+ run: (input) => {+ if (input.args.includes("read-tree") && input.args.includes("index.sparse=true"))+ freshReadTrees++;+ return liveProcess.run(input);+ },+ }),+ );const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-sparse-" });const { git } = yield* makeCheckpointFixture(driver, cwd);const write = Effect.fn(function* (name: string, contents: string) {@@ -680,11 +690,12 @@ it.effect.each("working outside\n",);}+ assert.strictEqual(freshReadTrees, indexState === "missing" ? 2 : 0);}).pipe(Effect.scoped, Effect.provide(layerGitContract)),);it.effect.each(- (["normal", "flags", "sparse"] as const).flatMap((indexMode) =>+ (["normal", "skip-worktree", "assume-unchanged", "sparse"] as const).flatMap((indexMode) =>[1, 3, 4096].map((chunkSize) => ({ indexMode, chunkSize })),),)(@@ -700,6 +711,7 @@ it.effect.each(yield* fs.writeFileString(path.join(cwd, ".gitattributes"), "stable filter=probe\n");yield* fs.writeFileString(path.join(cwd, "stable"), "unchanged\n");yield* fs.writeFileString(path.join(cwd, "z-skipped"), "original\n");+ yield* fs.writeFileString(path.join(cwd, "z-deleted"), "original\n");yield* fs.makeDirectory(path.join(cwd, "excluded"));yield* fs.writeFileString(path.join(cwd, "excluded/file"), "absent\n");yield* fs.writeFileString(@@ -707,19 +719,31 @@ it.effect.each('require("node:fs").appendFileSync(".git/reads", "read\\n"); process.stdin.pipe(process.stdout);',);yield* git(["config", "filter.probe.clean", "node .git/filter.cjs"]);+ yield* git(["config", "core.trustctime", "false"]);yield* fs.utimes(path.join(cwd, "stable"), 1_700_000_000, 1_700_000_000);+ yield* fs.utimes(path.join(cwd, "z-skipped"), 1_700_000_000, 1_700_000_000);yield* git(["add", "."]);yield* git(["commit", "-m", "inspection fixture"]);- if (indexMode === "flags") yield* git(["update-index", "--skip-worktree", "z-skipped"]);+ const flagged = indexMode === "skip-worktree" || indexMode === "assume-unchanged";+ if (flagged) {+ yield* git(["update-index", `--${indexMode}`, "z-skipped", "z-deleted"]);+ yield* git(["config", "core.ignorestat", "true"]);+ yield* fs.remove(path.join(cwd, "z-deleted"));+ }if (indexMode === "sparse")yield* git(["sparse-checkout", "set", "--cone", "--sparse-index", "included"]);yield* fs.writeFileString(path.join(cwd, "z-skipped"), "modified\n");+ // A flagged edit with the indexed size and mtime is invisible to Git's stat check.+ if (flagged) yield* fs.utimes(path.join(cwd, "z-skipped"), 1_700_000_000, 1_700_000_000);yield* fs.writeFileString(path.join(cwd, ".git/reads"), "");const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index"));+ let freshReadTrees = 0;const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(Effect.provideService(VcsProcess.VcsProcess, {- run: (input) =>- liveProcess.run(+ run: (input) => {+ if (input.args.includes("read-tree") && !input.args.includes("--reset"))+ freshReadTrees++;+ return liveProcess.run(input.args.includes("ls-files")? {...input,@@ -730,13 +754,19 @@ it.effect.each(},}: input,- ),+ );+ },}),);yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef });assert.strictEqual((yield* git(["show", `${checkpointRef}:z-skipped`])).stdout, "modified\n");- if (indexMode !== "flags")- assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/reads")), "");+ const files = (yield* git(["ls-tree", "-z", "--name-only", checkpointRef])).stdout.split(+ "\0",+ );+ if (flagged) assert.notInclude(files, "z-deleted");+ else assert.include(files, "z-deleted");+ assert.strictEqual(freshReadTrees, 0);+ assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/reads")), "");assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex);}).pipe(Effect.scoped, Effect.provide(layerGitContract)),);diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.tsindex db53eaf2be3..15392cb96ce 100644--- a/apps/server/src/vcs/GitVcsDriver.ts+++ b/apps/server/src/vcs/GitVcsDriver.ts@@ -444,6 +444,7 @@ export class GitVcsDriver extends Context.Service<>()("t3/vcs/GitVcsDriver") {}const WORKSPACE_FILES_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;+const FLAGGED_INDEX_ENTRIES_MAX_BYTES = 1024 * 1024;const CHECKPOINT_RECOVERY_MAX_CANDIDATES = 64;const CHECKPOINT_RECOVERY_TIMEOUT = "5 seconds";const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024;@@ -967,43 +968,60 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (});// read-tree can rewrite the index, so restore its racy timestamp afterward.yield* fileSystem.utimes(tempIndexPath, indexTime, indexTime);- let specialFlags = false;+ // Manually flagged entries hide edits from add -A, so they are re-added below+ // with no stat cache. Every other entry keeps its stat data and is not rehashed.+ let fallback = false;+ let flaggedBytes = 0;let recordStart = true;- let skipped = false;- let skippedRecord: Uint8Array[] = [];- const skippedPaths: string[] = [];+ let collect: "flagged" | "skipped" | undefined;+ let record: Uint8Array[] = [];+ const flagged: string[] = [];+ const skipped: string[] = [];yield* vcsProcess.run({operation,command: "git",cwd: input.cwd,- args: [...indexConfig, "ls-files", "--full-name", "--sparse", "-v", "-z"],+ args: [...indexConfig, "ls-files", "--full-name", "--sparse", "--stage", "-v", "-z"],env: commitEnv,maxOutputBytes: 4_096,outputMode: "truncate",- // Only tags and sparse skip-worktree paths matter; scan NUL-delimited records.+ // Only tags and flagged or skip-worktree records matter; scan NUL-delimited records.onStdoutChunk: (chunk) => {let start = 0;- while (start < chunk.length) {+ while (start < chunk.length && !fallback) {const end = chunk.indexOf(0, start);if (recordStart) {const tag = chunk[start]!;- skipped = tag === 83;- if ((tag >= 97 && tag <= 122) || (!sparseCheckout && tag === 83)) {- specialFlags = true;- }+ record = [];+ // Sparse skip-worktree bits are expected; only in-cone ones are manual.+ collect =+ tag === 83 && sparseCheckout+ ? "skipped"+ : (tag >= 97 && tag <= 122) || tag === 83+ ? "flagged"+ : undefined;+ // index-info cannot keep skip-worktree while dropping assume-unchanged.+ if (tag === 115 && sparseCheckout) fallback = true;}- if (skipped && sparseCheckout) {- skippedRecord.push(- Buffer.from(chunk.subarray(start, end === -1 ? undefined : end)),- );- if (end !== -1) {- const record = Buffer.concat(skippedRecord);- if (record.at(-1) !== 47) {- const name = record.subarray(2);- if (!NodeBuffer.isUtf8(name)) specialFlags = true;- else skippedPaths.push(name.toString("utf8"));+ if (collect !== undefined && !fallback) {+ const piece = chunk.subarray(start, end === -1 ? undefined : end);+ record.push(Buffer.from(piece));+ if (+ collect === "flagged" &&+ (flaggedBytes += piece.length) > FLAGGED_INDEX_ENTRIES_MAX_BYTES+ )+ fallback = true;+ if (end !== -1 && !fallback) {+ const full = Buffer.concat(record);+ record = [];+ // Sparse directory entries stay collapsed.+ if (collect === "flagged" || full.at(-1) !== 47) {+ // Drop the "<tag> " prefix, leaving index-info's "<mode> <oid> <stage>\t<path>".+ const entry = full.subarray(2);+ if (!NodeBuffer.isUtf8(entry)) fallback = true;+ else+ (collect === "flagged" ? flagged : skipped).push(entry.toString("utf8"));}- skippedRecord = [];}}recordStart = end !== -1;@@ -1012,20 +1030,46 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (}},});- if (skippedPaths.length > 0 && !specialFlags) {- const selected = yield* execute({+ if (fallback) return false;+ if (skipped.length > 0) {+ const paths = skipped.map((entry) => entry.slice(entry.indexOf("\t") + 1));+ const stdin = paths.join("\0") + "\0";+ // check-rules echoes a subset of its input, so the input size bounds its output.+ const rules = yield* execute({operation,cwd: input.cwd,args: [...indexConfig, "sparse-checkout", "check-rules", "-z"],- stdin: skippedPaths.join("\0") + "\0",+ stdin,+ env: commitEnv,+ maxOutputBytes: Buffer.byteLength(stdin),+ outputMode: "error",+ });+ const selected = new Set(rules.stdout.split("\0"));+ skipped.forEach((entry, index) => {+ if (selected.has(paths[index]!)) flagged.push(entry);+ });+ }+ if (flagged.length > 0) {+ // Flagged gitlinks keep the fresh-index behaviour rather than a guessed one.+ if (flagged.some((entry) => entry.startsWith("160000 "))) return false;+ yield* execute({+ operation,+ cwd: input.cwd,+ // core.ignorestat would mark these assume-unchanged, hiding edits from add -A.+ args: [+ ...indexConfig,+ "-c",+ "core.ignorestat=false",+ "update-index",+ "-z",+ "--index-info",+ ],+ stdin: flagged.join("\0") + "\0",env: commitEnv,- maxOutputBytes: 1,- outputMode: "truncate",});- specialFlags = selected.stdout.length > 0 || selected.stdoutTruncated;+ yield* fileSystem.utimes(tempIndexPath, indexTime, indexTime);}- // Sparse Git clears skip-worktree for present files. Manual flags still need a reset.- return !specialFlags;+ return true;}).pipe(Effect.orElseSucceed(() => false));if (!reusedIndex) {if (sparseCheckout) {