Skip to content
TypeScript

fix(server): checkpoints no longer rehash every file when one entry is skip-worktree or assume-unchanged (#23624)

Julius Marminge07a1e799d6dd
diff --git a/apps/server/src/vcs/GitVcsDriver.test.ts b/apps/server/src/vcs/GitVcsDriver.test.ts
index 9cb88072194..9262688fd10 100644
--- a/apps/server/src/vcs/GitVcsDriver.test.ts
+++ b/apps/server/src/vcs/GitVcsDriver.test.ts
@@ -597,7 +597,17 @@ it.effect.each(
Effect.gen(function* () {
const fs = yield* FileSystem.FileSystem;
const path = yield* Path.Path;
- const driver = yield* GitVcsDriver.makeVcsDriverShape();
+ const liveProcess = yield* VcsProcess.VcsProcess;
+ let freshReadTrees = 0;
+ const driver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
+ Effect.provideService(VcsProcess.VcsProcess, {
+ run: (input) => {
+ if (input.args.includes("read-tree") && input.args.includes("index.sparse=true"))
+ freshReadTrees++;
+ return liveProcess.run(input);
+ },
+ }),
+ );
const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-checkpoint-sparse-" });
const { git } = yield* makeCheckpointFixture(driver, cwd);
const write = Effect.fn(function* (name: string, contents: string) {
@@ -680,11 +690,12 @@ it.effect.each(
"working outside\n",
);
}
+ assert.strictEqual(freshReadTrees, indexState === "missing" ? 2 : 0);
}).pipe(Effect.scoped, Effect.provide(layerGitContract)),
);
it.effect.each(
- (["normal", "flags", "sparse"] as const).flatMap((indexMode) =>
+ (["normal", "skip-worktree", "assume-unchanged", "sparse"] as const).flatMap((indexMode) =>
[1, 3, 4096].map((chunkSize) => ({ indexMode, chunkSize })),
),
)(
@@ -700,6 +711,7 @@ it.effect.each(
yield* fs.writeFileString(path.join(cwd, ".gitattributes"), "stable filter=probe\n");
yield* fs.writeFileString(path.join(cwd, "stable"), "unchanged\n");
yield* fs.writeFileString(path.join(cwd, "z-skipped"), "original\n");
+ yield* fs.writeFileString(path.join(cwd, "z-deleted"), "original\n");
yield* fs.makeDirectory(path.join(cwd, "excluded"));
yield* fs.writeFileString(path.join(cwd, "excluded/file"), "absent\n");
yield* fs.writeFileString(
@@ -707,19 +719,31 @@ it.effect.each(
'require("node:fs").appendFileSync(".git/reads", "read\\n"); process.stdin.pipe(process.stdout);',
);
yield* git(["config", "filter.probe.clean", "node .git/filter.cjs"]);
+ yield* git(["config", "core.trustctime", "false"]);
yield* fs.utimes(path.join(cwd, "stable"), 1_700_000_000, 1_700_000_000);
+ yield* fs.utimes(path.join(cwd, "z-skipped"), 1_700_000_000, 1_700_000_000);
yield* git(["add", "."]);
yield* git(["commit", "-m", "inspection fixture"]);
- if (indexMode === "flags") yield* git(["update-index", "--skip-worktree", "z-skipped"]);
+ const flagged = indexMode === "skip-worktree" || indexMode === "assume-unchanged";
+ if (flagged) {
+ yield* git(["update-index", `--${indexMode}`, "z-skipped", "z-deleted"]);
+ yield* git(["config", "core.ignorestat", "true"]);
+ yield* fs.remove(path.join(cwd, "z-deleted"));
+ }
if (indexMode === "sparse")
yield* git(["sparse-checkout", "set", "--cone", "--sparse-index", "included"]);
yield* fs.writeFileString(path.join(cwd, "z-skipped"), "modified\n");
+ // A flagged edit with the indexed size and mtime is invisible to Git's stat check.
+ if (flagged) yield* fs.utimes(path.join(cwd, "z-skipped"), 1_700_000_000, 1_700_000_000);
yield* fs.writeFileString(path.join(cwd, ".git/reads"), "");
const originalIndex = yield* fs.readFile(path.join(cwd, ".git/index"));
+ let freshReadTrees = 0;
const captureDriver = yield* GitVcsDriver.makeVcsDriverShape().pipe(
Effect.provideService(VcsProcess.VcsProcess, {
- run: (input) =>
- liveProcess.run(
+ run: (input) => {
+ if (input.args.includes("read-tree") && !input.args.includes("--reset"))
+ freshReadTrees++;
+ return liveProcess.run(
input.args.includes("ls-files")
? {
...input,
@@ -730,13 +754,19 @@ it.effect.each(
},
}
: input,
- ),
+ );
+ },
}),
);
yield* captureDriver.checkpoints.captureCheckpoint({ cwd, checkpointRef });
assert.strictEqual((yield* git(["show", `${checkpointRef}:z-skipped`])).stdout, "modified\n");
- if (indexMode !== "flags")
- assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/reads")), "");
+ const files = (yield* git(["ls-tree", "-z", "--name-only", checkpointRef])).stdout.split(
+ "\0",
+ );
+ if (flagged) assert.notInclude(files, "z-deleted");
+ else assert.include(files, "z-deleted");
+ assert.strictEqual(freshReadTrees, 0);
+ assert.strictEqual(yield* fs.readFileString(path.join(cwd, ".git/reads")), "");
assert.deepEqual(yield* fs.readFile(path.join(cwd, ".git/index")), originalIndex);
}).pipe(Effect.scoped, Effect.provide(layerGitContract)),
);
diff --git a/apps/server/src/vcs/GitVcsDriver.ts b/apps/server/src/vcs/GitVcsDriver.ts
index db53eaf2be3..15392cb96ce 100644
--- a/apps/server/src/vcs/GitVcsDriver.ts
+++ b/apps/server/src/vcs/GitVcsDriver.ts
@@ -444,6 +444,7 @@ export class GitVcsDriver extends Context.Service<
>()("t3/vcs/GitVcsDriver") {}
const WORKSPACE_FILES_MAX_OUTPUT_BYTES = 16 * 1024 * 1024;
+const FLAGGED_INDEX_ENTRIES_MAX_BYTES = 1024 * 1024;
const CHECKPOINT_RECOVERY_MAX_CANDIDATES = 64;
const CHECKPOINT_RECOVERY_TIMEOUT = "5 seconds";
const GIT_CHECK_IGNORE_MAX_STDIN_BYTES = 256 * 1024;
@@ -967,43 +968,60 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
});
// read-tree can rewrite the index, so restore its racy timestamp afterward.
yield* fileSystem.utimes(tempIndexPath, indexTime, indexTime);
- let specialFlags = false;
+ // Manually flagged entries hide edits from add -A, so they are re-added below
+ // with no stat cache. Every other entry keeps its stat data and is not rehashed.
+ let fallback = false;
+ let flaggedBytes = 0;
let recordStart = true;
- let skipped = false;
- let skippedRecord: Uint8Array[] = [];
- const skippedPaths: string[] = [];
+ let collect: "flagged" | "skipped" | undefined;
+ let record: Uint8Array[] = [];
+ const flagged: string[] = [];
+ const skipped: string[] = [];
yield* vcsProcess.run({
operation,
command: "git",
cwd: input.cwd,
- args: [...indexConfig, "ls-files", "--full-name", "--sparse", "-v", "-z"],
+ args: [...indexConfig, "ls-files", "--full-name", "--sparse", "--stage", "-v", "-z"],
env: commitEnv,
maxOutputBytes: 4_096,
outputMode: "truncate",
- // Only tags and sparse skip-worktree paths matter; scan NUL-delimited records.
+ // Only tags and flagged or skip-worktree records matter; scan NUL-delimited records.
onStdoutChunk: (chunk) => {
let start = 0;
- while (start < chunk.length) {
+ while (start < chunk.length && !fallback) {
const end = chunk.indexOf(0, start);
if (recordStart) {
const tag = chunk[start]!;
- skipped = tag === 83;
- if ((tag >= 97 && tag <= 122) || (!sparseCheckout && tag === 83)) {
- specialFlags = true;
- }
+ record = [];
+ // Sparse skip-worktree bits are expected; only in-cone ones are manual.
+ collect =
+ tag === 83 && sparseCheckout
+ ? "skipped"
+ : (tag >= 97 && tag <= 122) || tag === 83
+ ? "flagged"
+ : undefined;
+ // index-info cannot keep skip-worktree while dropping assume-unchanged.
+ if (tag === 115 && sparseCheckout) fallback = true;
}
- if (skipped && sparseCheckout) {
- skippedRecord.push(
- Buffer.from(chunk.subarray(start, end === -1 ? undefined : end)),
- );
- if (end !== -1) {
- const record = Buffer.concat(skippedRecord);
- if (record.at(-1) !== 47) {
- const name = record.subarray(2);
- if (!NodeBuffer.isUtf8(name)) specialFlags = true;
- else skippedPaths.push(name.toString("utf8"));
+ if (collect !== undefined && !fallback) {
+ const piece = chunk.subarray(start, end === -1 ? undefined : end);
+ record.push(Buffer.from(piece));
+ if (
+ collect === "flagged" &&
+ (flaggedBytes += piece.length) > FLAGGED_INDEX_ENTRIES_MAX_BYTES
+ )
+ fallback = true;
+ if (end !== -1 && !fallback) {
+ const full = Buffer.concat(record);
+ record = [];
+ // Sparse directory entries stay collapsed.
+ if (collect === "flagged" || full.at(-1) !== 47) {
+ // Drop the "<tag> " prefix, leaving index-info's "<mode> <oid> <stage>\t<path>".
+ const entry = full.subarray(2);
+ if (!NodeBuffer.isUtf8(entry)) fallback = true;
+ else
+ (collect === "flagged" ? flagged : skipped).push(entry.toString("utf8"));
}
- skippedRecord = [];
}
}
recordStart = end !== -1;
@@ -1012,20 +1030,46 @@ export const makeVcsDriverShape = Effect.fn("makeGitVcsDriverShape")(function* (
}
},
});
- if (skippedPaths.length > 0 && !specialFlags) {
- const selected = yield* execute({
+ if (fallback) return false;
+ if (skipped.length > 0) {
+ const paths = skipped.map((entry) => entry.slice(entry.indexOf("\t") + 1));
+ const stdin = paths.join("\0") + "\0";
+ // check-rules echoes a subset of its input, so the input size bounds its output.
+ const rules = yield* execute({
operation,
cwd: input.cwd,
args: [...indexConfig, "sparse-checkout", "check-rules", "-z"],
- stdin: skippedPaths.join("\0") + "\0",
+ stdin,
+ env: commitEnv,
+ maxOutputBytes: Buffer.byteLength(stdin),
+ outputMode: "error",
+ });
+ const selected = new Set(rules.stdout.split("\0"));
+ skipped.forEach((entry, index) => {
+ if (selected.has(paths[index]!)) flagged.push(entry);
+ });
+ }
+ if (flagged.length > 0) {
+ // Flagged gitlinks keep the fresh-index behaviour rather than a guessed one.
+ if (flagged.some((entry) => entry.startsWith("160000 "))) return false;
+ yield* execute({
+ operation,
+ cwd: input.cwd,
+ // core.ignorestat would mark these assume-unchanged, hiding edits from add -A.
+ args: [
+ ...indexConfig,
+ "-c",
+ "core.ignorestat=false",
+ "update-index",
+ "-z",
+ "--index-info",
+ ],
+ stdin: flagged.join("\0") + "\0",
env: commitEnv,
- maxOutputBytes: 1,
- outputMode: "truncate",
});
- specialFlags = selected.stdout.length > 0 || selected.stdoutTruncated;
+ yield* fileSystem.utimes(tempIndexPath, indexTime, indexTime);
}
- // Sparse Git clears skip-worktree for present files. Manual flags still need a reset.
- return !specialFlags;
+ return true;
}).pipe(Effect.orElseSucceed(() => false));
if (!reusedIndex) {
if (sparseCheckout) {