Skip to content
TypeScript

fix(server): slow forge CLI calls no longer delay checkpoints and turn starts (#28431)

Julius Marminged5afb1a0dd94
diff --git a/apps/server/src/orchestration-v2/RunFinalizationService.test.ts b/apps/server/src/orchestration-v2/RunFinalizationService.test.ts
index 38a18192531..b51ec8fb80d 100644
--- a/apps/server/src/orchestration-v2/RunFinalizationService.test.ts
+++ b/apps/server/src/orchestration-v2/RunFinalizationService.test.ts
@@ -5,8 +5,12 @@ import {
ThreadId,
type OrchestrationV2ThreadShell,
} from "@t3tools/contracts";
+import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
+import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
+import * as TestClock from "effect/testing/TestClock";
import * as PullRequestService from "../pullRequest/PullRequestService.ts";
import * as VcsStatusBroadcaster from "../vcs/VcsStatusBroadcaster.ts";
@@ -93,6 +97,7 @@ it.effect.each(
).map((scenario) => [scenario.label, scenario] as const),
)("%s", ([, scenario]) => {
const refreshed: string[] = [];
+ const lookedUp = Deferred.makeUnsafe<void>();
const threadId = ThreadId.make("thread-pr-refresh");
const runId = RunId.make("completed-run");
const layer = RunFinalization.layerObserver.pipe(
@@ -114,11 +119,11 @@ it.effect.each(
}),
refreshStatus: () =>
Effect.die("turn completion must preserve known PRs and lookup backoff"),
- refreshPullRequestStatus: (cwd) =>
- Effect.sync(() => {
- refreshed.push(cwd);
- return null;
- }),
+ // Requested synchronously by refresh; the lookup itself runs in the background.
+ refreshPullRequestStatus: (cwd) => {
+ refreshed.push(cwd);
+ return Deferred.succeed(lookedUp, undefined).pipe(Effect.as(null));
+ },
}),
Layer.mock(ProjectionStore.ProjectionStoreV2)({
getThreadShell: () =>
@@ -134,6 +139,105 @@ it.effect.each(
return Effect.gen(function* () {
const observer = yield* RunFinalization.RunFinalizationObserver;
yield* observer.refresh({ cwd: "/repo", threadId, runId });
+ if (scenario.expected.length > 0) yield* Deferred.await(lookedUp);
assert.deepEqual(refreshed, [...scenario.expected]);
}).pipe(Effect.provide(layer));
});
+
+it.effect("returns from the post-turn refresh without waiting for the PR lookup", () => {
+ const threadId = ThreadId.make("thread-slow-forge");
+ const runId = RunId.make("completed-run");
+ const layer = RunFinalization.layerObserver.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }),
+ Layer.mock(PullRequestService.PullRequestService)({
+ refreshAfterTurn: () => Effect.void,
+ }),
+ Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({
+ refreshLocalStatus: () =>
+ Effect.succeed({
+ isRepo: true,
+ hasPrimaryRemote: true,
+ isDefaultRef: false,
+ refName: "feature",
+ hasWorkingTreeChanges: false,
+ workingTree: { files: [], insertions: 0, deletions: 0 },
+ }),
+ refreshPullRequestStatus: () => Effect.never,
+ }),
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({
+ getThreadShell: () =>
+ Effect.succeed({
+ id: threadId,
+ branch: "feature",
+ activeRunId: null,
+ } as OrchestrationV2ThreadShell),
+ }),
+ ),
+ ),
+ );
+ return Effect.gen(function* () {
+ const observer = yield* RunFinalization.RunFinalizationObserver;
+ const refresh = yield* observer
+ .refresh({ cwd: "/repo", threadId, runId })
+ .pipe(Effect.timeoutOption("1 minute"), Effect.forkChild({ startImmediately: true }));
+ yield* TestClock.adjust("1 minute");
+ const result = yield* Fiber.join(refresh);
+ assert.isTrue(
+ Option.isSome(result),
+ "turn finalization waited on the forge PR lookup, blocking the thread's next turn",
+ );
+ }).pipe(Effect.provide(layer));
+});
+
+it.effect("replaces a still-running PR lookup when a newer turn finishes", () => {
+ const threadId = ThreadId.make("thread-back-to-back");
+ const runId = RunId.make("completed-run");
+ const firstInterrupted = Deferred.makeUnsafe<void>();
+ const secondLookedUp = Deferred.makeUnsafe<void>();
+ let lookups = 0;
+ const layer = RunFinalization.layerObserver.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ Layer.mock(WorkspaceEntries.WorkspaceEntries)({ refresh: () => Effect.void }),
+ Layer.mock(PullRequestService.PullRequestService)({
+ refreshAfterTurn: () => Effect.void,
+ }),
+ Layer.mock(VcsStatusBroadcaster.VcsStatusBroadcaster)({
+ refreshLocalStatus: () =>
+ Effect.succeed({
+ isRepo: true,
+ hasPrimaryRemote: true,
+ isDefaultRef: false,
+ refName: "feature",
+ hasWorkingTreeChanges: false,
+ workingTree: { files: [], insertions: 0, deletions: 0 },
+ }),
+ refreshPullRequestStatus: () =>
+ ++lookups === 1
+ ? Effect.never.pipe(
+ Effect.onInterrupt(() => Deferred.succeed(firstInterrupted, undefined)),
+ )
+ : Deferred.succeed(secondLookedUp, undefined).pipe(Effect.as(null)),
+ }),
+ Layer.mock(ProjectionStore.ProjectionStoreV2)({
+ getThreadShell: () =>
+ Effect.succeed({
+ id: threadId,
+ branch: "feature",
+ activeRunId: null,
+ } as OrchestrationV2ThreadShell),
+ }),
+ ),
+ ),
+ );
+ return Effect.gen(function* () {
+ const observer = yield* RunFinalization.RunFinalizationObserver;
+ yield* observer.refresh({ cwd: "/repo", threadId, runId });
+ yield* observer.refresh({ cwd: "/repo", threadId, runId });
+ yield* Deferred.await(firstInterrupted);
+ yield* Deferred.await(secondLookedUp);
+ assert.strictEqual(lookups, 2);
+ }).pipe(Effect.provide(layer));
+});
diff --git a/apps/server/src/orchestration-v2/RunFinalizationService.ts b/apps/server/src/orchestration-v2/RunFinalizationService.ts
index 40882033ba5..3775e63f0e7 100644
--- a/apps/server/src/orchestration-v2/RunFinalizationService.ts
+++ b/apps/server/src/orchestration-v2/RunFinalizationService.ts
@@ -1,6 +1,7 @@
import { CheckpointScopeId, ProjectId, RunId, ThreadId } from "@t3tools/contracts";
import * as Context from "effect/Context";
import * as Effect from "effect/Effect";
+import * as FiberMap from "effect/FiberMap";
import * as Layer from "effect/Layer";
import * as Schema from "effect/Schema";
@@ -96,6 +97,7 @@ export const layerObserver = Layer.effect(
const vcsStatus = yield* VcsStatusBroadcaster.VcsStatusBroadcaster;
const projections = yield* ProjectionStore.ProjectionStoreV2;
const pullRequests = yield* PullRequestService.PullRequestService;
+ const pullRequestLookups = yield* FiberMap.make<string>();
return {
refreshAfterTurn: pullRequests.refreshAfterTurn,
refresh: Effect.fn("RunFinalizationService.refresh")(
@@ -108,6 +110,8 @@ export const layerObserver = Layer.effect(
const thread = yield* projections.getThreadShell(threadId);
if (!thread || thread.branch !== local.refName) return;
if (thread.activeRunId !== null && thread.activeRunId !== runId) return;
+ // The forge CLI can take minutes, so the next turn must not wait for it. A newer
+ // turn's lookup replaces a running one instead of queueing behind the cwd's lock.
yield* vcsStatus.refreshPullRequestStatus(cwd).pipe(
Effect.catch((error) =>
Effect.logWarning("failed to refresh pull request status after run completion", {
@@ -116,6 +120,7 @@ export const layerObserver = Layer.effect(
detail: error.message,
}),
),
+ FiberMap.run(pullRequestLookups, cwd),
);
},
(effect, { cwd }) =>
diff --git a/apps/server/src/vcs/VcsProcess.test.ts b/apps/server/src/vcs/VcsProcess.test.ts
index 212415171bd..199ded803ee 100644
--- a/apps/server/src/vcs/VcsProcess.test.ts
+++ b/apps/server/src/vcs/VcsProcess.test.ts
@@ -8,10 +8,12 @@ import * as Cause from "effect/Cause";
import * as Fiber from "effect/Fiber";
import * as FileSystem from "effect/FileSystem";
import * as Layer from "effect/Layer";
+import * as Option from "effect/Option";
import * as Path from "effect/Path";
import * as Queue from "effect/Queue";
import * as Ref from "effect/Ref";
import * as Schema from "effect/Schema";
+import * as TestClock from "effect/testing/TestClock";
import { ChildProcessSpawner } from "effect/process";
import { VcsProcessExitError, VcsProcessSpawnError } from "@t3tools/contracts";
@@ -316,6 +318,60 @@ describe("VcsProcess.run", () => {
}),
);
+ it.effect("leaves git a slot while slow forge CLI processes are running", () =>
+ Effect.gen(function* () {
+ const gate = yield* Deferred.make<void>();
+ const service = yield* VcsProcess.make.pipe(
+ Effect.provideService(
+ ProcessRunner.ProcessRunner,
+ ProcessRunner.ProcessRunner.of({
+ run: (input) =>
+ Effect.gen(function* () {
+ if (input.command !== "git") yield* Deferred.await(gate);
+ return {
+ stdout: "",
+ stderr: "",
+ code: ChildProcessSpawner.ExitCode(0),
+ timedOut: false,
+ stdoutTruncated: false,
+ stderrTruncated: false,
+ stdoutInvalidUtf8: false,
+ stderrInvalidUtf8: false,
+ };
+ }),
+ }),
+ ),
+ );
+
+ const forgeRuns = yield* Effect.forEach(Array.from({ length: 12 }), () =>
+ service
+ .run({
+ operation: "synthetic.forge.listChangeRequests",
+ command: "tea",
+ args: ["pulls", "list"],
+ cwd: "/workspace",
+ })
+ .pipe(Effect.forkChild({ startImmediately: true })),
+ );
+ const checkpoint = yield* service
+ .run({
+ ...baseInput,
+ operation: VcsProcess.CHECKPOINT_CAPTURE_OPERATION,
+ args: ["rev-parse", "HEAD"],
+ })
+ .pipe(Effect.timeoutOption("1 second"), Effect.forkChild({ startImmediately: true }));
+ yield* TestClock.adjust("1 second");
+
+ const gitResult = yield* Fiber.join(checkpoint);
+ yield* Deferred.succeed(gate, undefined);
+ yield* Fiber.joinAll(forgeRuns);
+ expect(
+ Option.isSome(gitResult),
+ "git waited behind forge CLI runs that held every VCS process slot",
+ ).toBe(true);
+ }),
+ );
+
it.effect("keeps git from opening a folder laid out as a bare repository", () =>
Effect.gen(function* () {
const fileSystem = yield* FileSystem.FileSystem;
diff --git a/apps/server/src/vcs/VcsProcess.ts b/apps/server/src/vcs/VcsProcess.ts
index 00218397481..c668a35e992 100644
--- a/apps/server/src/vcs/VcsProcess.ts
+++ b/apps/server/src/vcs/VcsProcess.ts
@@ -65,6 +65,8 @@ const DEFAULT_TIMEOUT_MS = 30_000;
const DEFAULT_MAX_OUTPUT_BYTES = 1_000_000;
const OUTPUT_TRUNCATED_MARKER = "\n\n[truncated]";
const VCS_PROCESS_CONCURRENCY = 8;
+// Forge CLIs can take seconds per call; cap them together so git, checkpoints included, keeps slots.
+const NON_GIT_PROCESS_CONCURRENCY = VCS_PROCESS_CONCURRENCY / 2;
/** Shown to users when a Git command fails because another process holds the index lock. */
export const GIT_INDEX_LOCKED_DETAIL =
@@ -183,6 +185,7 @@ export const make = Effect.gen(function* () {
const processRunner = yield* ProcessRunner.ProcessRunner;
const hostPlatform = yield* HostProcess.Platform;
const vcsProcesses = yield* Semaphore.make(VCS_PROCESS_CONCURRENCY);
+ const nonGitProcesses = yield* Semaphore.make(NON_GIT_PROCESS_CONCURRENCY);
const keyedProcesses = new Map<string, Semaphore.Semaphore>();
const keyedPermit = (concurrency: NonNullable<VcsProcessInput["concurrency"]>) => {
let semaphore = keyedProcesses.get(concurrency.key);
@@ -365,9 +368,8 @@ export const make = Effect.gen(function* () {
}),
);
}
- return yield* input.concurrency === undefined
- ? bounded
- : keyedPermit(input.concurrency)(bounded);
+ const shared = input.command === "git" ? bounded : nonGitProcesses.withPermits(1)(bounded);
+ return yield* input.concurrency === undefined ? shared : keyedPermit(input.concurrency)(shared);
});
return VcsProcess.of({ run });