Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

Truncate oversized git diffs instead of failing (#1499)

Julius Marminge9e60597174dc
diff --git a/apps/server/src/checkpointing/Layers/CheckpointStore.test.ts b/apps/server/src/checkpointing/Layers/CheckpointStore.test.ts
new file mode 100644
index 000000000..c430dbbde
--- /dev/null
+++ b/apps/server/src/checkpointing/Layers/CheckpointStore.test.ts
@@ -0,0 +1,122 @@
+import path from "node:path";
+
+import * as NodeServices from "@effect/platform-node/NodeServices";
+import { it } from "@effect/vitest";
+import { Effect, FileSystem, Layer, PlatformError, Scope } from "effect";
+import { describe, expect } from "vitest";
+
+import { checkpointRefForThreadTurn } from "../Utils.ts";
+import { CheckpointStoreLive } from "./CheckpointStore.ts";
+import { CheckpointStore } from "../Services/CheckpointStore.ts";
+import { GitCoreLive } from "../../git/Layers/GitCore.ts";
+import { GitCore } from "../../git/Services/GitCore.ts";
+import { GitCommandError } from "../../git/Errors.ts";
+import { ServerConfig } from "../../config.ts";
+import { ThreadId } from "@t3tools/contracts";
+
+const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {
+ prefix: "t3-checkpoint-store-test-",
+});
+const GitCoreTestLayer = GitCoreLive.pipe(
+ Layer.provide(ServerConfigLayer),
+ Layer.provide(NodeServices.layer),
+);
+const CheckpointStoreTestLayer = CheckpointStoreLive.pipe(
+ Layer.provide(GitCoreTestLayer),
+ Layer.provide(NodeServices.layer),
+);
+const TestLayer = Layer.mergeAll(NodeServices.layer, GitCoreTestLayer, CheckpointStoreTestLayer);
+
+function makeTmpDir(
+ prefix = "checkpoint-store-test-",
+): Effect.Effect<string, PlatformError.PlatformError, FileSystem.FileSystem | Scope.Scope> {
+ return Effect.gen(function* () {
+ const fileSystem = yield* FileSystem.FileSystem;
+ return yield* fileSystem.makeTempDirectoryScoped({ prefix });
+ });
+}
+
+function writeTextFile(
+ filePath: string,
+ contents: string,
+): Effect.Effect<void, PlatformError.PlatformError, FileSystem.FileSystem> {
+ return Effect.gen(function* () {
+ const fileSystem = yield* FileSystem.FileSystem;
+ yield* fileSystem.writeFileString(filePath, contents);
+ });
+}
+
+function git(
+ cwd: string,
+ args: ReadonlyArray<string>,
+): Effect.Effect<string, GitCommandError, GitCore> {
+ return Effect.gen(function* () {
+ const gitCore = yield* GitCore;
+ const result = yield* gitCore.execute({
+ operation: "CheckpointStore.test.git",
+ cwd,
+ args,
+ timeoutMs: 10_000,
+ });
+ return result.stdout.trim();
+ });
+}
+
+function initRepoWithCommit(
+ cwd: string,
+): Effect.Effect<
+ void,
+ GitCommandError | PlatformError.PlatformError,
+ GitCore | FileSystem.FileSystem
+> {
+ return Effect.gen(function* () {
+ const core = yield* GitCore;
+ yield* core.initRepo({ cwd });
+ yield* git(cwd, ["config", "user.email", "[email protected]"]);
+ yield* git(cwd, ["config", "user.name", "Test"]);
+ yield* writeTextFile(path.join(cwd, "README.md"), "# test\n");
+ yield* git(cwd, ["add", "."]);
+ yield* git(cwd, ["commit", "-m", "initial commit"]);
+ });
+}
+
+function buildLargeText(lineCount = 20_000): string {
+ return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`)
+ .join("\n")
+ .concat("\n");
+}
+
+it.layer(TestLayer)("CheckpointStoreLive", (it) => {
+ describe("diffCheckpoints", () => {
+ it.effect("returns full oversized checkpoint diffs without truncation", () =>
+ Effect.gen(function* () {
+ const tmp = yield* makeTmpDir();
+ yield* initRepoWithCommit(tmp);
+ const checkpointStore = yield* CheckpointStore;
+ const threadId = ThreadId.makeUnsafe("thread-checkpoint-store");
+ const fromCheckpointRef = checkpointRefForThreadTurn(threadId, 0);
+ const toCheckpointRef = checkpointRefForThreadTurn(threadId, 1);
+
+ yield* checkpointStore.captureCheckpoint({
+ cwd: tmp,
+ checkpointRef: fromCheckpointRef,
+ });
+ yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText());
+ yield* checkpointStore.captureCheckpoint({
+ cwd: tmp,
+ checkpointRef: toCheckpointRef,
+ });
+
+ const diff = yield* checkpointStore.diffCheckpoints({
+ cwd: tmp,
+ fromCheckpointRef,
+ toCheckpointRef,
+ });
+
+ expect(diff).toContain("diff --git");
+ expect(diff).not.toContain("[truncated]");
+ expect(diff).toContain("+line 19999");
+ }),
+ );
+ });
+});
diff --git a/apps/server/src/git/Layers/GitCore.test.ts b/apps/server/src/git/Layers/GitCore.test.ts
index dc97b9364..547a69e7e 100644
--- a/apps/server/src/git/Layers/GitCore.test.ts
+++ b/apps/server/src/git/Layers/GitCore.test.ts
@@ -132,6 +132,12 @@ function commitWithDate(
});
}
+function buildLargeText(lineCount = 20_000): string {
+ return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`)
+ .join("\n")
+ .concat("\n");
+}
+
// ── Tests ──
it.layer(TestLayer)("git integration", (it) => {
@@ -1670,6 +1676,40 @@ it.layer(TestLayer)("git integration", (it) => {
}),
);
+ it.effect("prepareCommitContext truncates oversized staged patches instead of failing", () =>
+ Effect.gen(function* () {
+ const tmp = yield* makeTmpDir();
+ yield* initRepoWithCommit(tmp);
+ const core = yield* GitCore;
+
+ yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText());
+
+ const context = yield* core.prepareCommitContext(tmp);
+ expect(context).not.toBeNull();
+ expect(context!.stagedSummary).toContain("README.md");
+ expect(context!.stagedPatch).toContain("[truncated]");
+ }),
+ );
+
+ it.effect("readRangeContext truncates oversized diff patches instead of failing", () =>
+ Effect.gen(function* () {
+ const tmp = yield* makeTmpDir();
+ const { initialBranch } = yield* initRepoWithCommit(tmp);
+ const core = yield* GitCore;
+
+ yield* core.createBranch({ cwd: tmp, branch: "feature/large-range-context" });
+ yield* core.checkoutBranch({ cwd: tmp, branch: "feature/large-range-context" });
+ yield* writeTextFile(path.join(tmp, "large.txt"), buildLargeText());
+ yield* git(tmp, ["add", "large.txt"]);
+ yield* git(tmp, ["commit", "-m", "Add large range context"]);
+
+ const rangeContext = yield* core.readRangeContext(tmp, initialBranch);
+ expect(rangeContext.commitSummary).toContain("Add large range context");
+ expect(rangeContext.diffSummary).toContain("large.txt");
+ expect(rangeContext.diffPatch).toContain("[truncated]");
+ }),
+ );
+
it.effect("pushes with upstream setup and then skips when up to date", () =>
Effect.gen(function* () {
const tmp = yield* makeTmpDir();
diff --git a/apps/server/src/git/Layers/GitCore.ts b/apps/server/src/git/Layers/GitCore.ts
index 8bb584422..fcb2f9c58 100644
--- a/apps/server/src/git/Layers/GitCore.ts
+++ b/apps/server/src/git/Layers/GitCore.ts
@@ -32,6 +32,11 @@ import { decodeJsonResult } from "@t3tools/shared/schemaJson";
const DEFAULT_TIMEOUT_MS = 30_000;
const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;
+const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]";
+const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000;
+const RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES = 19_000;
+const RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES = 19_000;
+const RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES = 59_000;
const STATUS_UPSTREAM_REFRESH_INTERVAL = Duration.seconds(15);
const STATUS_UPSTREAM_REFRESH_TIMEOUT = Duration.seconds(5);
const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048;
@@ -53,6 +58,8 @@ interface ExecuteGitOptions {
timeoutMs?: number | undefined;
allowNonZeroExit?: boolean | undefined;
fallbackErrorMessage?: string | undefined;
+ maxOutputBytes?: number | undefined;
+ truncateOutputAtMaxBytes?: boolean | undefined;
progress?: ExecuteGitProgress | undefined;
}
@@ -439,12 +446,14 @@ const collectOutput = Effect.fn(function* <E>(
input: Pick<ExecuteGitInput, "operation" | "cwd" | "args">,
stream: Stream.Stream<Uint8Array, E>,
maxOutputBytes: number,
+ truncateOutputAtMaxBytes: boolean,
onLine: ((line: string) => Effect.Effect<void, never>) | undefined,
): Effect.fn.Return<string, GitCommandError> {
const decoder = new TextDecoder();
let bytes = 0;
let text = "";
let lineBuffer = "";
+ let truncated = false;
const emitCompleteLines = (flush: boolean) =>
Effect.gen(function* () {
@@ -469,8 +478,11 @@ const collectOutput = Effect.fn(function* <E>(
yield* Stream.runForEach(stream, (chunk) =>
Effect.gen(function* () {
- bytes += chunk.byteLength;
- if (bytes > maxOutputBytes) {
+ if (truncateOutputAtMaxBytes && truncated) {
+ return;
+ }
+ const nextBytes = bytes + chunk.byteLength;
+ if (!truncateOutputAtMaxBytes && nextBytes > maxOutputBytes) {
return yield* new GitCommandError({
operation: input.operation,
command: quoteGitCommand(input.args),
@@ -478,18 +490,26 @@ const collectOutput = Effect.fn(function* <E>(
detail: `${quoteGitCommand(input.args)} output exceeded ${maxOutputBytes} bytes and was truncated.`,
});
}
- const decoded = decoder.decode(chunk, { stream: true });
+
+ const chunkToDecode =
+ truncateOutputAtMaxBytes && nextBytes > maxOutputBytes
+ ? chunk.subarray(0, Math.max(0, maxOutputBytes - bytes))
+ : chunk;
+ bytes += chunkToDecode.byteLength;
+ truncated = truncateOutputAtMaxBytes && nextBytes > maxOutputBytes;
+
+ const decoded = decoder.decode(chunkToDecode, { stream: !truncated });
text += decoded;
lineBuffer += decoded;
yield* emitCompleteLines(false);
}),
).pipe(Effect.mapError(toGitCommandError(input, "output stream failed.")));
- const remainder = decoder.decode();
+ const remainder = truncated ? "" : decoder.decode();
text += remainder;
lineBuffer += remainder;
yield* emitCompleteLines(true);
- return text;
+ return truncated ? `${text}${OUTPUT_TRUNCATED_MARKER}` : text;
});
export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"] }) =>
@@ -511,6 +531,7 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
} as const;
const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;
const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;
+ const truncateOutputAtMaxBytes = input.truncateOutputAtMaxBytes ?? false;
const commandEffect = Effect.gen(function* () {
const trace2Monitor = yield* createTrace2Monitor(commandInput, input.progress).pipe(
@@ -537,12 +558,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
commandInput,
child.stdout,
maxOutputBytes,
+ truncateOutputAtMaxBytes,
input.progress?.onStdoutLine,
),
collectOutput(
commandInput,
child.stderr,
maxOutputBytes,
+ truncateOutputAtMaxBytes,
input.progress?.onStderrLine,
),
child.exitCode.pipe(
@@ -603,6 +626,10 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
args,
allowNonZeroExit: true,
...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),
+ ...(options.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),
+ ...(options.truncateOutputAtMaxBytes !== undefined
+ ? { truncateOutputAtMaxBytes: options.truncateOutputAtMaxBytes }
+ : {}),
...(options.progress ? { progress: options.progress } : {}),
}).pipe(
Effect.flatMap((result) => {
@@ -647,6 +674,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
Effect.map((result) => result.stdout),
);
+ const runGitStdoutWithOptions = (
+ operation: string,
+ cwd: string,
+ args: readonly string[],
+ options: ExecuteGitOptions = {},
+ ): Effect.Effect<string, GitCommandError> =>
+ executeGit(operation, cwd, args, options).pipe(Effect.map((result) => result.stdout));
+
const branchExists = (cwd: string, branch: string): Effect.Effect<boolean, GitCommandError> =>
executeGit(
"GitCore.branchExists",
@@ -1162,12 +1197,15 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
return null;
}
- const stagedPatch = yield* runGitStdout("GitCore.prepareCommitContext.stagedPatch", cwd, [
- "diff",
- "--cached",
- "--patch",
- "--minimal",
- ]);
+ const stagedPatch = yield* runGitStdoutWithOptions(
+ "GitCore.prepareCommitContext.stagedPatch",
+ cwd,
+ ["diff", "--cached", "--patch", "--minimal"],
+ {
+ maxOutputBytes: PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES,
+ truncateOutputAtMaxBytes: true,
+ },
+ );
return {
stagedSummary,
@@ -1363,14 +1401,33 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"
const range = `${baseBranch}..HEAD`;
const [commitSummary, diffSummary, diffPatch] = yield* Effect.all(
[
- runGitStdout("GitCore.readRangeContext.log", cwd, ["log", "--oneline", range]),
- runGitStdout("GitCore.readRangeContext.diffStat", cwd, ["diff", "--stat", range]),
- runGitStdout("GitCore.readRangeContext.diffPatch", cwd, [
- "diff",
- "--patch",
- "--minimal",
- range,
- ]),
+ runGitStdoutWithOptions(
+ "GitCore.readRangeContext.log",
+ cwd,
+ ["log", "--oneline", range],
+ {
+ maxOutputBytes: RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES,
+ truncateOutputAtMaxBytes: true,
+ },
+ ),
+ runGitStdoutWithOptions(
+ "GitCore.readRangeContext.diffStat",
+ cwd,
+ ["diff", "--stat", range],
+ {
+ maxOutputBytes: RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES,
+ truncateOutputAtMaxBytes: true,
+ },
+ ),
+ runGitStdoutWithOptions(
+ "GitCore.readRangeContext.diffPatch",
+ cwd,
+ ["diff", "--patch", "--minimal", range],
+ {
+ maxOutputBytes: RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES,
+ truncateOutputAtMaxBytes: true,
+ },
+ ),
],
{ concurrency: "unbounded" },
);
diff --git a/apps/server/src/git/Services/GitCore.ts b/apps/server/src/git/Services/GitCore.ts
index b74c52689..f1a4e065c 100644
--- a/apps/server/src/git/Services/GitCore.ts
+++ b/apps/server/src/git/Services/GitCore.ts
@@ -32,6 +32,7 @@ export interface ExecuteGitInput {
readonly allowNonZeroExit?: boolean;
readonly timeoutMs?: number;
readonly maxOutputBytes?: number;
+ readonly truncateOutputAtMaxBytes?: boolean;
readonly progress?: ExecuteGitProgress;
}