TypeScript
fix(server): slow forge CLI calls no longer delay checkpoints and turn starts (#28431)
Julius Marminged5afb1a0dd94
diff --git a/apps/server/src/orchestration-v2/RunFinalizationService.test.ts b/apps/server/src/orchestration-v2/RunFinalizationService.test.tsindex 38a18192531..b51ec8fb80d 100644--- a/apps/server/src/orchestration-v2/RunFinalizationService.test.ts+++ b/apps/server/src/orchestration-v2/RunFinalizationService.test.ts@@ -5,8 +5,12 @@ import {ThreadId,type OrchestrationV2ThreadShell,} from "@t3tools/contracts";+import * as Deferred from "effect/Deferred";import * as Effect from "effect/Effect";+import * as Fiber from "effect/Fiber";import * as Layer from "effect/Layer";+import * as Option from "effect/Option";+import * as TestClock from "effect/testing/TestClock";import * as PullRequestService from "../pullRequest/PullRequestService.ts";import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts";@@ -93,6 +97,7 @@ it.effect.each().map((scenario) => [scenario.label, scenario] as const),)("%s", ([, scenario]) => {const refreshed: string[] = [];+ const lookedUp = Deferred.makeUnsafe<void>();const threadId = ThreadId.make("thread-pr-refresh");const runId = RunId.make("completed-run");const layer = RunFinalization.layerObserver.pipe(@@ -114,11 +119,11 @@ it.effect.each(}),refreshStatus: () =>Effect.die("turn completion must preserve known PRs and lookup backoff"),- refreshPullRequestStatus: (cwd) =>- Effect.sync(() => {- refreshed.push(cwd);- return null;- }),+ // Requested synchronously by refresh; the lookup itself runs in the background.+ refreshPullRequestStatus: (cwd) => {+ refreshed.push(cwd);+ return Deferred.succeed(lookedUp, undefined).pipe(Effect.as(null));+ },}),Layer.mock(ProjectionStore.ProjectionStoreV2)({getThreadShell: () =>@@ -134,6 +139,105 @@ it.effect.each(return Effect.gen(function* () {const observer = yield* RunFinalization.RunFinalizationObserver;yield* observer.refresh({ cwd: "/repo", threadId, runId });+ if (scenario.expected.length > 0) yield* Deferred.await(lookedUp);assert.deepEqual(refreshed, [...scenario.expected]);}).pipe(Effect.provide(layer));});++it.effect("returns from the post-turn refresh without waiting for the PR lookup", () => {+ const threadId = ThreadId.make("thread-slow-forge");+ const runId = RunId.make("completed-run");+ const layer = RunFinalization.layerObserver.pipe(+ Layer.provide(+ Layer.mergeAll(+ Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }),+ Layer.mock(PullRequestService.PullRequestService)({+ refreshAfterTurn: () => Effect.void,+ }),+ Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({+ refreshLocalStatus: () =>+ Effect.succeed({+ isRepo: true,+ hasPrimaryRemote: true,+ isDefaultRef: false,+ refName: "feature",+ hasWorkingTreeChanges: false,+ workingTree: { files: [], insertions: 0, deletions: 0 },+ }),+ refreshPullRequestStatus: () => Effect.never,+ }),+ Layer.mock(ProjectionStore.ProjectionStoreV2)({+ getThreadShell: () =>+ Effect.succeed({+ id: threadId,+ branch: "feature",+ activeRunId: null,+ } as OrchestrationV2ThreadShell),+ }),+ ),+ ),+ );+ return Effect.gen(function* () {+ const observer = yield* RunFinalization.RunFinalizationObserver;+ const refresh = yield* observer+ .refresh({ cwd: "/repo", threadId, runId })+ .pipe(Effect.timeoutOption("1 minute"), Effect.forkChild({ startImmediately: true }));+ yield* TestClock.adjust("1 minute");+ const result = yield* Fiber.join(refresh);+ assert.isTrue(+ Option.isSome(result),+ "turn finalization waited on the forge PR lookup, blocking the thread's next turn",+ );+ }).pipe(Effect.provide(layer));+});++it.effect("replaces a still-running PR lookup when a newer turn finishes", () => {+ const threadId = ThreadId.make("thread-back-to-back");+ const runId = RunId.make("completed-run");+ const firstInterrupted = Deferred.makeUnsafe<void>();+ const secondLookedUp = Deferred.makeUnsafe<void>();+ let lookups = 0;+ const layer = RunFinalization.layerObserver.pipe(+ Layer.provide(+ Layer.mergeAll(+ Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }),+ Layer.mock(PullRequestService.PullRequestService)({+ refreshAfterTurn: () => Effect.void,+ }),+ Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({+ refreshLocalStatus: () =>+ Effect.succeed({+ isRepo: true,+ hasPrimaryRemote: true,+ isDefaultRef: false,+ refName: "feature",+ hasWorkingTreeChanges: false,+ workingTree: { files: [], insertions: 0, deletions: 0 },+ }),+ refreshPullRequestStatus: () =>+ ++lookups === 1+ ? Effect.never.pipe(+ Effect.onInterrupt(() => Deferred.succeed(firstInterrupted, undefined)),+ )+ : Deferred.succeed(secondLookedUp, undefined).pipe(Effect.as(null)),+ }),+ Layer.mock(ProjectionStore.ProjectionStoreV2)({+ getThreadShell: () =>+ Effect.succeed({+ id: threadId,+ branch: "feature",+ activeRunId: null,+ } as OrchestrationV2ThreadShell),+ }),+ ),+ ),+ );+ return Effect.gen(function* () {+ const observer = yield* RunFinalization.RunFinalizationObserver;+ yield* observer.refresh({ cwd: "/repo", threadId, runId });+ yield* observer.refresh({ cwd: "/repo", threadId, runId });+ yield* Deferred.await(firstInterrupted);+ yield* Deferred.await(secondLookedUp);+ assert.strictEqual(lookups, 2);+ }).pipe(Effect.provide(layer));+});diff --git a/apps/server/src/orchestration-v2/RunFinalizationService.ts b/apps/server/src/orchestration-v2/RunFinalizationService.tsindex 40882033ba5..3775e63f0e7 100644--- a/apps/server/src/orchestration-v2/RunFinalizationService.ts+++ b/apps/server/src/orchestration-v2/RunFinalizationService.ts@@ -1,6 +1,7 @@import { CheckpointScopeId, ProjectId, RunId, ThreadId } from "@t3tools/contracts";import * as Context from "effect/Context";import * as Effect from "effect/Effect";+import * as FiberMap from "effect/FiberMap";import * as Layer from "effect/Layer";import * as Schema from "effect/Schema";@@ -96,6 +97,7 @@ export const layerObserver = Layer.effect(const vcsStatus = yield* VcsStatusBroadcaster.VcsStatusBroadcaster;const projections = yield* ProjectionStore.ProjectionStoreV2;const pullRequests = yield* PullRequestService.PullRequestService;+ const pullRequestLookups = yield* FiberMap.make<string>();return {refreshAfterTurn: pullRequests.refreshAfterTurn,refresh: Effect.fn("RunFinalizationService.refresh")(@@ -108,6 +110,8 @@ export const layerObserver = Layer.effect(const thread = yield* projections.getThreadShell(threadId);if (!thread || thread.branch !== local.refName) return;if (thread.activeRunId !== null && thread.activeRunId !== runId) return;+ // The forge CLI can take minutes, so the next turn must not wait for it. A newer+ // turn's lookup replaces a running one instead of queueing behind the cwd's lock.yield* vcsStatus.refreshPullRequestStatus(cwd).pipe(Effect.catch((error) =>Effect.logWarning("failed to refresh pull request status after run completion", {@@ -116,6 +120,7 @@ export const layerObserver = Layer.effect(detail: error.message,}),),+ FiberMap.run(pullRequestLookups, cwd),);},(effect, { cwd }) =>diff --git a/apps/server/src/vcs/VcsProcess.test.ts b/apps/server/src/vcs/VcsProcess.test.tsindex 212415171bd..199ded803ee 100644--- a/apps/server/src/vcs/VcsProcess.test.ts+++ b/apps/server/src/vcs/VcsProcess.test.ts@@ -8,10 +8,12 @@ import * as Cause from "effect/Cause";import * as Fiber from "effect/Fiber";import * as FileSystem from "effect/FileSystem";import * as Layer from "effect/Layer";+import * as Option from "effect/Option";import * as Path from "effect/Path";import * as Queue from "effect/Queue";import * as Ref from "effect/Ref";import * as Schema from "effect/Schema";+import * as TestClock from "effect/testing/TestClock";import { ChildProcessSpawner } from "effect/process";import { VcsProcessExitError, VcsProcessSpawnError } from "@t3tools/contracts";@@ -316,6 +318,60 @@ describe("VcsProcess.run", () => {}),);+ it.effect("leaves git a slot while slow forge CLI processes are running", () =>+ Effect.gen(function* () {+ const gate = yield* Deferred.make<void>();+ const service = yield* VcsProcess.make.pipe(+ Effect.provideService(+ ProcessRunner.ProcessRunner,+ ProcessRunner.ProcessRunner.of({+ run: (input) =>+ Effect.gen(function* () {+ if (input.command !== "git") yield* Deferred.await(gate);+ return {+ stdout: "",+ stderr: "",+ code: ChildProcessSpawner.ExitCode(0),+ timedOut: false,+ stdoutTruncated: false,+ stderrTruncated: false,+ stdoutInvalidUtf8: false,+ stderrInvalidUtf8: false,+ };+ }),+ }),+ ),+ );++ const forgeRuns = yield* Effect.forEach(Array.from({ length: 12 }), () =>+ service+ .run({+ operation: "synthetic.forge.listChangeRequests",+ command: "tea",+ args: ["pulls", "list"],+ cwd: "/workspace",+ })+ .pipe(Effect.forkChild({ startImmediately: true })),+ );+ const checkpoint = yield* service+ .run({+ ...baseInput,+ operation: VcsProcess.CHECKPOINT_CAPTURE_OPERATION,+ args: ["rev-parse", "HEAD"],+ })+ .pipe(Effect.timeoutOption("1 second"), Effect.forkChild({ startImmediately: true }));+ yield* TestClock.adjust("1 second");++ const gitResult = yield* Fiber.join(checkpoint);+ yield* Deferred.succeed(gate, undefined);+ yield* Fiber.joinAll(forgeRuns);+ expect(+ Option.isSome(gitResult),+ "git waited behind forge CLI runs that held every VCS process slot",+ ).toBe(true);+ }),+ );+it.effect("keeps git from opening a folder laid out as a bare repository", () =>Effect.gen(function* () {const fileSystem = yield* FileSystem.FileSystem;diff --git a/apps/server/src/vcs/VcsProcess.ts b/apps/server/src/vcs/VcsProcess.tsindex 00218397481..c668a35e992 100644--- a/apps/server/src/vcs/VcsProcess.ts+++ b/apps/server/src/vcs/VcsProcess.ts@@ -65,6 +65,8 @@ const DEFAULT_TIMEOUT_MS = 30_000;const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]";const VCS_PROCESS_CONCURRENCY = 8;+// Forge CLIs can take seconds per call; cap them together so git, checkpoints included, keeps slots.+const NON_GIT_PROCESS_CONCURRENCY = VCS_PROCESS_CONCURRENCY / 2;/** Shown to users when a Git command fails because another process holds the index lock. */export const GIT_INDEX_LOCKED_DETAIL =@@ -183,6 +185,7 @@ export const make = Effect.gen(function* () {const processRunner = yield* ProcessRunner.ProcessRunner;const hostPlatform = yield* HostProcess.Platform;const vcsProcesses = yield* Semaphore.make(VCS_PROCESS_CONCURRENCY);+ const nonGitProcesses = yield* Semaphore.make(NON_GIT_PROCESS_CONCURRENCY);const keyedProcesses = new Map<string, Semaphore.Semaphore>();const keyedPermit = (concurrency: NonNullable<VcsProcessInput["concurrency"]>) => {let semaphore = keyedProcesses.get(concurrency.key);@@ -365,9 +368,8 @@ export const make = Effect.gen(function* () {}),);}- return yield* input.concurrency === undefined- ? bounded- : keyedPermit(input.concurrency)(bounded);+ const shared = input.command === "git" ? bounded : nonGitProcesses.withPermits(1)(bounded);+ return yield* input.concurrency === undefined ? shared : keyedPermit(input.concurrency)(shared);});return VcsProcess.of({ run });