Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
Truncate oversized git diffs instead of failing (#1499)
Julius Marminge9e60597174dc
diff --git a/apps/server/src/checkpointing/Layers/CheckpointStore.test.ts b/apps/server/src/checkpointing/Layers/CheckpointStore.test.tsnew file mode 100644index 000000000..c430dbbde--- /dev/null+++ b/apps/server/src/checkpointing/Layers/CheckpointStore.test.ts@@ -0,0 +1,122 @@+import path from "node:path";++import * as NodeServices from "@effect/platform-node/NodeServices";+import { it } from "@effect/vitest";+import { Effect, FileSystem, Layer, PlatformError, Scope } from "effect";+import { describe, expect } from "vitest";++import { checkpointRefForThreadTurn } from "../Utils.ts";+import { CheckpointStoreLive } from "./CheckpointStore.ts";+import { CheckpointStore } from "../Services/CheckpointStore.ts";+import { GitCoreLive } from "../../git/Layers/GitCore.ts";+import { GitCore } from "../../git/Services/GitCore.ts";+import { GitCommandError } from "../../git/Errors.ts";+import { ServerConfig } from "../../config.ts";+import { ThreadId } from "@t3tools/contracts";++const ServerConfigLayer = ServerConfig.layerTest(process.cwd(), {+ prefix: "t3-checkpoint-store-test-",+});+const GitCoreTestLayer = GitCoreLive.pipe(+ Layer.provide(ServerConfigLayer),+ Layer.provide(NodeServices.layer),+);+const CheckpointStoreTestLayer = CheckpointStoreLive.pipe(+ Layer.provide(GitCoreTestLayer),+ Layer.provide(NodeServices.layer),+);+const TestLayer = Layer.mergeAll(NodeServices.layer, GitCoreTestLayer, CheckpointStoreTestLayer);++function makeTmpDir(+ prefix = "checkpoint-store-test-",+): Effect.Effect<string, PlatformError.PlatformError, FileSystem.FileSystem | Scope.Scope> {+ return Effect.gen(function* () {+ const fileSystem = yield* FileSystem.FileSystem;+ return yield* fileSystem.makeTempDirectoryScoped({ prefix });+ });+}++function writeTextFile(+ filePath: string,+ contents: string,+): Effect.Effect<void, PlatformError.PlatformError, FileSystem.FileSystem> {+ return Effect.gen(function* () {+ const fileSystem = yield* FileSystem.FileSystem;+ yield* fileSystem.writeFileString(filePath, contents);+ });+}++function git(+ cwd: string,+ args: ReadonlyArray<string>,+): Effect.Effect<string, GitCommandError, GitCore> {+ return Effect.gen(function* () {+ const gitCore = yield* GitCore;+ const result = yield* gitCore.execute({+ operation: "CheckpointStore.test.git",+ cwd,+ args,+ timeoutMs: 10_000,+ });+ return result.stdout.trim();+ });+}++function initRepoWithCommit(+ cwd: string,+): Effect.Effect<+ void,+ GitCommandError | PlatformError.PlatformError,+ GitCore | FileSystem.FileSystem+> {+ return Effect.gen(function* () {+ const core = yield* GitCore;+ yield* core.initRepo({ cwd });+ yield* git(cwd, ["config", "user.email", "[email protected]"]);+ yield* git(cwd, ["config", "user.name", "Test"]);+ yield* writeTextFile(path.join(cwd, "README.md"), "# test\n");+ yield* git(cwd, ["add", "."]);+ yield* git(cwd, ["commit", "-m", "initial commit"]);+ });+}++function buildLargeText(lineCount = 20_000): string {+ return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`)+ .join("\n")+ .concat("\n");+}++it.layer(TestLayer)("CheckpointStoreLive", (it) => {+ describe("diffCheckpoints", () => {+ it.effect("returns full oversized checkpoint diffs without truncation", () =>+ Effect.gen(function* () {+ const tmp = yield* makeTmpDir();+ yield* initRepoWithCommit(tmp);+ const checkpointStore = yield* CheckpointStore;+ const threadId = ThreadId.makeUnsafe("thread-checkpoint-store");+ const fromCheckpointRef = checkpointRefForThreadTurn(threadId, 0);+ const toCheckpointRef = checkpointRefForThreadTurn(threadId, 1);++ yield* checkpointStore.captureCheckpoint({+ cwd: tmp,+ checkpointRef: fromCheckpointRef,+ });+ yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText());+ yield* checkpointStore.captureCheckpoint({+ cwd: tmp,+ checkpointRef: toCheckpointRef,+ });++ const diff = yield* checkpointStore.diffCheckpoints({+ cwd: tmp,+ fromCheckpointRef,+ toCheckpointRef,+ });++ expect(diff).toContain("diff --git");+ expect(diff).not.toContain("[truncated]");+ expect(diff).toContain("+line 19999");+ }),+ );+ });+});diff --git a/apps/server/src/git/Layers/GitCore.test.ts b/apps/server/src/git/Layers/GitCore.test.tsindex dc97b9364..547a69e7e 100644--- a/apps/server/src/git/Layers/GitCore.test.ts+++ b/apps/server/src/git/Layers/GitCore.test.ts@@ -132,6 +132,12 @@ function commitWithDate(});}+function buildLargeText(lineCount = 20_000): string {+ return Array.from({ length: lineCount }, (_, index) => `line ${String(index).padStart(5, "0")}`)+ .join("\n")+ .concat("\n");+}+// ── Tests ──it.layer(TestLayer)("git integration", (it) => {@@ -1670,6 +1676,40 @@ it.layer(TestLayer)("git integration", (it) => {}),);+ it.effect("prepareCommitContext truncates oversized staged patches instead of failing", () =>+ Effect.gen(function* () {+ const tmp = yield* makeTmpDir();+ yield* initRepoWithCommit(tmp);+ const core = yield* GitCore;++ yield* writeTextFile(path.join(tmp, "README.md"), buildLargeText());++ const context = yield* core.prepareCommitContext(tmp);+ expect(context).not.toBeNull();+ expect(context!.stagedSummary).toContain("README.md");+ expect(context!.stagedPatch).toContain("[truncated]");+ }),+ );++ it.effect("readRangeContext truncates oversized diff patches instead of failing", () =>+ Effect.gen(function* () {+ const tmp = yield* makeTmpDir();+ const { initialBranch } = yield* initRepoWithCommit(tmp);+ const core = yield* GitCore;++ yield* core.createBranch({ cwd: tmp, branch: "feature/large-range-context" });+ yield* core.checkoutBranch({ cwd: tmp, branch: "feature/large-range-context" });+ yield* writeTextFile(path.join(tmp, "large.txt"), buildLargeText());+ yield* git(tmp, ["add", "large.txt"]);+ yield* git(tmp, ["commit", "-m", "Add large range context"]);++ const rangeContext = yield* core.readRangeContext(tmp, initialBranch);+ expect(rangeContext.commitSummary).toContain("Add large range context");+ expect(rangeContext.diffSummary).toContain("large.txt");+ expect(rangeContext.diffPatch).toContain("[truncated]");+ }),+ );+it.effect("pushes with upstream setup and then skips when up to date", () =>Effect.gen(function* () {const tmp = yield* makeTmpDir();diff --git a/apps/server/src/git/Layers/GitCore.ts b/apps/server/src/git/Layers/GitCore.tsindex 8bb584422..fcb2f9c58 100644--- a/apps/server/src/git/Layers/GitCore.ts+++ b/apps/server/src/git/Layers/GitCore.ts@@ -32,6 +32,11 @@ import { decodeJsonResult } from "@t3tools/shared/schemaJson";const DEFAULT_TIMEOUT_MS = 30_000;const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;+const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]";+const PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES = 49_000;+const RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES = 19_000;+const RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES = 19_000;+const RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES = 59_000;const STATUS_UPSTREAM_REFRESH_INTERVAL = Duration.seconds(15);const STATUS_UPSTREAM_REFRESH_TIMEOUT = Duration.seconds(5);const STATUS_UPSTREAM_REFRESH_CACHE_CAPACITY = 2_048;@@ -53,6 +58,8 @@ interface ExecuteGitOptions {timeoutMs?: number | undefined;allowNonZeroExit?: boolean | undefined;fallbackErrorMessage?: string | undefined;+ maxOutputBytes?: number | undefined;+ truncateOutputAtMaxBytes?: boolean | undefined;progress?: ExecuteGitProgress | undefined;}@@ -439,12 +446,14 @@ const collectOutput = Effect.fn(function* <E>(input: Pick<ExecuteGitInput, "operation" | "cwd" | "args">,stream: Stream.Stream<Uint8Array, E>,maxOutputBytes: number,+ truncateOutputAtMaxBytes: boolean,onLine: ((line: string) => Effect.Effect<void, never>) | undefined,): Effect.fn.Return<string, GitCommandError> {const decoder = new TextDecoder();let bytes = 0;let text = "";let lineBuffer = "";+ let truncated = false;const emitCompleteLines = (flush: boolean) =>Effect.gen(function* () {@@ -469,8 +478,11 @@ const collectOutput = Effect.fn(function* <E>(yield* Stream.runForEach(stream, (chunk) =>Effect.gen(function* () {- bytes += chunk.byteLength;- if (bytes > maxOutputBytes) {+ if (truncateOutputAtMaxBytes && truncated) {+ return;+ }+ const nextBytes = bytes + chunk.byteLength;+ if (!truncateOutputAtMaxBytes && nextBytes > maxOutputBytes) {return yield* new GitCommandError({operation: input.operation,command: quoteGitCommand(input.args),@@ -478,18 +490,26 @@ const collectOutput = Effect.fn(function* <E>(detail: `${quoteGitCommand(input.args)} output exceeded ${maxOutputBytes} bytes and was truncated.`,});}- const decoded = decoder.decode(chunk, { stream: true });++ const chunkToDecode =+ truncateOutputAtMaxBytes && nextBytes > maxOutputBytes+ ? chunk.subarray(0, Math.max(0, maxOutputBytes - bytes))+ : chunk;+ bytes += chunkToDecode.byteLength;+ truncated = truncateOutputAtMaxBytes && nextBytes > maxOutputBytes;++ const decoded = decoder.decode(chunkToDecode, { stream: !truncated });text += decoded;lineBuffer += decoded;yield* emitCompleteLines(false);}),).pipe(Effect.mapError(toGitCommandError(input, "output stream failed.")));- const remainder = decoder.decode();+ const remainder = truncated ? "" : decoder.decode();text += remainder;lineBuffer += remainder;yield* emitCompleteLines(true);- return text;+ return truncated ? `${text}${OUTPUT_TRUNCATED_MARKER}` : text;});export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"] }) =>@@ -511,6 +531,7 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"} as const;const timeoutMs = input.timeoutMs ?? DEFAULT_TIMEOUT_MS;const maxOutputBytes = input.maxOutputBytes ?? DEFAULT_MAX_OUTPUT_BYTES;+ const truncateOutputAtMaxBytes = input.truncateOutputAtMaxBytes ?? false;const commandEffect = Effect.gen(function* () {const trace2Monitor = yield* createTrace2Monitor(commandInput, input.progress).pipe(@@ -537,12 +558,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"commandInput,child.stdout,maxOutputBytes,+ truncateOutputAtMaxBytes,input.progress?.onStdoutLine,),collectOutput(commandInput,child.stderr,maxOutputBytes,+ truncateOutputAtMaxBytes,input.progress?.onStderrLine,),child.exitCode.pipe(@@ -603,6 +626,10 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"args,allowNonZeroExit: true,...(options.timeoutMs !== undefined ? { timeoutMs: options.timeoutMs } : {}),+ ...(options.maxOutputBytes !== undefined ? { maxOutputBytes: options.maxOutputBytes } : {}),+ ...(options.truncateOutputAtMaxBytes !== undefined+ ? { truncateOutputAtMaxBytes: options.truncateOutputAtMaxBytes }+ : {}),...(options.progress ? { progress: options.progress } : {}),}).pipe(Effect.flatMap((result) => {@@ -647,6 +674,14 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"Effect.map((result) => result.stdout),);+ const runGitStdoutWithOptions = (+ operation: string,+ cwd: string,+ args: readonly string[],+ options: ExecuteGitOptions = {},+ ): Effect.Effect<string, GitCommandError> =>+ executeGit(operation, cwd, args, options).pipe(Effect.map((result) => result.stdout));+const branchExists = (cwd: string, branch: string): Effect.Effect<boolean, GitCommandError> =>executeGit("GitCore.branchExists",@@ -1162,12 +1197,15 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"return null;}- const stagedPatch = yield* runGitStdout("GitCore.prepareCommitContext.stagedPatch", cwd, [- "diff",- "--cached",- "--patch",- "--minimal",- ]);+ const stagedPatch = yield* runGitStdoutWithOptions(+ "GitCore.prepareCommitContext.stagedPatch",+ cwd,+ ["diff", "--cached", "--patch", "--minimal"],+ {+ maxOutputBytes: PREPARED_COMMIT_PATCH_MAX_OUTPUT_BYTES,+ truncateOutputAtMaxBytes: true,+ },+ );return {stagedSummary,@@ -1363,14 +1401,33 @@ export const makeGitCore = (options?: { executeOverride?: GitCoreShape["execute"const range = `${baseBranch}..HEAD`;const [commitSummary, diffSummary, diffPatch] = yield* Effect.all([- runGitStdout("GitCore.readRangeContext.log", cwd, ["log", "--oneline", range]),- runGitStdout("GitCore.readRangeContext.diffStat", cwd, ["diff", "--stat", range]),- runGitStdout("GitCore.readRangeContext.diffPatch", cwd, [- "diff",- "--patch",- "--minimal",- range,- ]),+ runGitStdoutWithOptions(+ "GitCore.readRangeContext.log",+ cwd,+ ["log", "--oneline", range],+ {+ maxOutputBytes: RANGE_COMMIT_SUMMARY_MAX_OUTPUT_BYTES,+ truncateOutputAtMaxBytes: true,+ },+ ),+ runGitStdoutWithOptions(+ "GitCore.readRangeContext.diffStat",+ cwd,+ ["diff", "--stat", range],+ {+ maxOutputBytes: RANGE_DIFF_SUMMARY_MAX_OUTPUT_BYTES,+ truncateOutputAtMaxBytes: true,+ },+ ),+ runGitStdoutWithOptions(+ "GitCore.readRangeContext.diffPatch",+ cwd,+ ["diff", "--patch", "--minimal", range],+ {+ maxOutputBytes: RANGE_DIFF_PATCH_MAX_OUTPUT_BYTES,+ truncateOutputAtMaxBytes: true,+ },+ ),],{ concurrency: "unbounded" },);diff --git a/apps/server/src/git/Services/GitCore.ts b/apps/server/src/git/Services/GitCore.tsindex b74c52689..f1a4e065c 100644--- a/apps/server/src/git/Services/GitCore.ts+++ b/apps/server/src/git/Services/GitCore.ts@@ -32,6 +32,7 @@ export interface ExecuteGitInput {readonly allowNonZeroExit?: boolean;readonly timeoutMs?: number;readonly maxOutputBytes?: number;+ readonly truncateOutputAtMaxBytes?: boolean;readonly progress?: ExecuteGitProgress;}