Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
Back off VCS remote refresh failures (#2686)
Julius Marminge4120e94598c7
diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts b/apps/server/src/vcs/VcsStatusBroadcaster.test.tsindex ccadf9a92..d47dda29b 100644--- a/apps/server/src/vcs/VcsStatusBroadcaster.test.ts+++ b/apps/server/src/vcs/VcsStatusBroadcaster.test.ts@@ -310,6 +310,29 @@ describe("VcsStatusBroadcaster", () => {}).pipe(Effect.provide(makeTestLayer(state)));});+ it("backs off remote refresh failures exponentially and honors larger configured intervals", () => {+ assert.equal(+ Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(1, Duration.seconds(1))),+ 30_000,+ );+ assert.equal(+ Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(2, Duration.seconds(1))),+ 60_000,+ );+ assert.equal(+ Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(3, Duration.seconds(1))),+ 120_000,+ );+ assert.equal(+ Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(1, Duration.minutes(5))),+ 300_000,+ );+ assert.equal(+ Duration.toMillis(VcsStatusBroadcaster.remoteRefreshFailureDelay(20, Duration.seconds(1))),+ 900_000,+ );+ });+it.effect("stops the remote poller after the last stream subscriber disconnects", () => {const state = {currentLocalStatus: baseLocalStatus,diff --git a/apps/server/src/vcs/VcsStatusBroadcaster.ts b/apps/server/src/vcs/VcsStatusBroadcaster.tsindex c93b72917..40cdcf2c8 100644--- a/apps/server/src/vcs/VcsStatusBroadcaster.ts+++ b/apps/server/src/vcs/VcsStatusBroadcaster.ts@@ -7,6 +7,7 @@ import * as FileSystem from "effect/FileSystem";import * as Layer from "effect/Layer";import * as PubSub from "effect/PubSub";import * as Ref from "effect/Ref";+import * as Schedule from "effect/Schedule";import * as Scope from "effect/Scope";import * as Stream from "effect/Stream";import * as SynchronizedRef from "effect/SynchronizedRef";@@ -23,6 +24,8 @@ import { mergeGitStatusParts } from "@t3tools/shared/git";import * as GitWorkflowService from "../git/GitWorkflowService.ts";const DEFAULT_VCS_STATUS_REFRESH_INTERVAL = Duration.seconds(30);+const VCS_STATUS_REFRESH_FAILURE_BASE_DELAY = Duration.seconds(30);+const VCS_STATUS_REFRESH_FAILURE_MAX_DELAY = Duration.minutes(15);interface VcsStatusChange {readonly cwd: string;@@ -48,6 +51,20 @@ interface StreamStatusOptions {readonly automaticRemoteRefreshInterval?: Effect.Effect<Duration.Duration, never>;}+export function remoteRefreshFailureDelay(+ consecutiveFailures: number,+ configuredInterval: Duration.Duration,+) {+ const exponent = Math.max(0, consecutiveFailures - 1);+ const backoffMs =+ Duration.toMillis(VCS_STATUS_REFRESH_FAILURE_BASE_DELAY) * Math.pow(2, exponent);+ const cappedBackoff = Duration.min(+ Duration.millis(backoffMs),+ VCS_STATUS_REFRESH_FAILURE_MAX_DELAY,+ );+ return Duration.max(configuredInterval, cappedBackoff);+}+export interface VcsStatusBroadcasterShape {readonly getStatus: (input: VcsStatusInput,@@ -241,32 +258,46 @@ export const layer = Layer.effect(cwd: string,automaticRemoteRefreshInterval: Effect.Effect<Duration.Duration, never>,) => {- const logRefreshFailure = (error: GitManagerServiceError) =>- Effect.logWarning("VCS remote status refresh failed", {- cwd,- detail: error.message,+ return Effect.gen(function* () {+ const consecutiveFailuresRef = yield* Ref.make(0);+ const refreshRemoteStatusIfEnabled = Effect.gen(function* () {+ const configuredInterval = yield* automaticRemoteRefreshInterval;+ const activeInterval = Duration.isZero(configuredInterval)+ ? DEFAULT_VCS_STATUS_REFRESH_INTERVAL+ : configuredInterval;+ if (Duration.isZero(configuredInterval)) {+ return activeInterval;+ }++ const exit = yield* refreshRemoteStatus(cwd).pipe(Effect.exit);+ if (Exit.isSuccess(exit)) {+ yield* Ref.set(consecutiveFailuresRef, 0);+ return activeInterval;+ }++ const consecutiveFailures = yield* Ref.updateAndGet(+ consecutiveFailuresRef,+ (count) => count + 1,+ );+ const nextDelay = remoteRefreshFailureDelay(consecutiveFailures, activeInterval);+ yield* Effect.logWarning("VCS remote status refresh failed", {+ cwd,+ detail: exit.cause.toString(),+ consecutiveFailures,+ nextDelayMs: Duration.toMillis(nextDelay),+ });+ return nextDelay;});- const refreshRemoteStatusIfEnabled = automaticRemoteRefreshInterval.pipe(- Effect.flatMap((interval) =>- Duration.isZero(interval) ? Effect.void : refreshRemoteStatus(cwd).pipe(Effect.asVoid),- ),- );- const sleepForConfiguredInterval = automaticRemoteRefreshInterval.pipe(- Effect.flatMap((interval) =>- Effect.sleep(Duration.isZero(interval) ? DEFAULT_VCS_STATUS_REFRESH_INTERVAL : interval),- ),- );- return refreshRemoteStatusIfEnabled.pipe(- Effect.catch(logRefreshFailure),- Effect.andThen(- Effect.forever(- sleepForConfiguredInterval.pipe(- Effect.andThen(refreshRemoteStatusIfEnabled.pipe(Effect.catch(logRefreshFailure))),+ return yield* refreshRemoteStatusIfEnabled.pipe(+ Effect.repeat(+ Schedule.identity<Duration.Duration>().pipe(+ Schedule.addDelay((delay) => Effect.succeed(delay)),),),- ),- );+ Effect.asVoid,+ );+ });};const retainRemotePoller = Effect.fn("VcsStatusBroadcaster.retainRemotePoller")(function* (