Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.
Return replay retry state from orchestration recovery (#1728)
Julius Marminge6de4b47ecb5f
diff --git a/apps/web/src/orchestrationRecovery.test.ts b/apps/web/src/orchestrationRecovery.test.tsindex fbdbea4ee..b04622cdb 100644--- a/apps/web/src/orchestrationRecovery.test.ts+++ b/apps/web/src/orchestrationRecovery.test.ts@@ -1,6 +1,9 @@import { describe, expect, it } from "vitest";-import { createOrchestrationRecoveryCoordinator } from "./orchestrationRecovery";+import {+ createOrchestrationRecoveryCoordinator,+ deriveReplayRetryDecision,+} from "./orchestrationRecovery";describe("createOrchestrationRecoveryCoordinator", () => {it("defers live events until bootstrap completes and then requests replay", () => {@@ -59,10 +62,13 @@ describe("createOrchestrationRecoveryCoordinator", () => {coordinator.classifyDomainEvent(7);coordinator.markEventBatchApplied([{ sequence: 4 }, { sequence: 5 }, { sequence: 6 }]);- expect(coordinator.completeReplayRecovery()).toBe(true);+ expect(coordinator.completeReplayRecovery()).toEqual({+ replayMadeProgress: true,+ shouldReplay: true,+ });});- it("does not immediately replay again when replay returns no new events", () => {+ it("retries replay when no progress was made but higher live sequences were observed", () => {const coordinator = createOrchestrationRecoveryCoordinator();coordinator.beginSnapshotRecovery("bootstrap");@@ -70,7 +76,10 @@ describe("createOrchestrationRecoveryCoordinator", () => {coordinator.classifyDomainEvent(5);coordinator.beginReplayRecovery("sequence-gap");- expect(coordinator.completeReplayRecovery()).toBe(false);+ expect(coordinator.completeReplayRecovery()).toEqual({+ replayMadeProgress: false,+ shouldReplay: true,+ });expect(coordinator.getState()).toMatchObject({latestSequence: 3,highestObservedSequence: 5,@@ -79,6 +88,19 @@ describe("createOrchestrationRecoveryCoordinator", () => {});});+ it("does not request another replay when a replay made no progress and nothing newer was observed", () => {+ const coordinator = createOrchestrationRecoveryCoordinator();++ coordinator.beginSnapshotRecovery("bootstrap");+ coordinator.completeSnapshotRecovery(3);+ coordinator.beginReplayRecovery("sequence-gap");++ expect(coordinator.completeReplayRecovery()).toEqual({+ replayMadeProgress: false,+ shouldReplay: false,+ });+ });+it("marks replay failure as unbootstrapped so snapshot fallback is recovery-only", () => {const coordinator = createOrchestrationRecoveryCoordinator();@@ -131,3 +153,154 @@ describe("createOrchestrationRecoveryCoordinator", () => {});});});++describe("deriveReplayRetryDecision", () => {+ it("retries immediately when replay made progress", () => {+ expect(+ deriveReplayRetryDecision({+ previousTracker: {+ attempts: 2,+ latestSequence: 3,+ highestObservedSequence: 5,+ },+ completion: {+ replayMadeProgress: true,+ shouldReplay: true,+ },+ recoveryState: {+ latestSequence: 5,+ highestObservedSequence: 5,+ },+ baseDelayMs: 100,+ maxNoProgressRetries: 3,+ }),+ ).toEqual({+ shouldRetry: true,+ delayMs: 0,+ tracker: null,+ });+ });++ it("caps no-progress retries for the same frontier", () => {+ const first = deriveReplayRetryDecision({+ previousTracker: null,+ completion: {+ replayMadeProgress: false,+ shouldReplay: true,+ },+ recoveryState: {+ latestSequence: 3,+ highestObservedSequence: 5,+ },+ baseDelayMs: 100,+ maxNoProgressRetries: 3,+ });++ const second = deriveReplayRetryDecision({+ previousTracker: first.tracker,+ completion: {+ replayMadeProgress: false,+ shouldReplay: true,+ },+ recoveryState: {+ latestSequence: 3,+ highestObservedSequence: 5,+ },+ baseDelayMs: 100,+ maxNoProgressRetries: 3,+ });++ const third = deriveReplayRetryDecision({+ previousTracker: second.tracker,+ completion: {+ replayMadeProgress: false,+ shouldReplay: true,+ },+ recoveryState: {+ latestSequence: 3,+ highestObservedSequence: 5,+ },+ baseDelayMs: 100,+ maxNoProgressRetries: 3,+ });++ const fourth = deriveReplayRetryDecision({+ previousTracker: third.tracker,+ completion: {+ replayMadeProgress: false,+ shouldReplay: true,+ },+ recoveryState: {+ latestSequence: 3,+ highestObservedSequence: 5,+ },+ baseDelayMs: 100,+ maxNoProgressRetries: 3,+ });++ expect(first).toEqual({+ shouldRetry: true,+ delayMs: 100,+ tracker: {+ attempts: 1,+ latestSequence: 3,+ highestObservedSequence: 5,+ },+ });+ expect(second).toEqual({+ shouldRetry: true,+ delayMs: 200,+ tracker: {+ attempts: 2,+ latestSequence: 3,+ highestObservedSequence: 5,+ },+ });+ expect(third).toEqual({+ shouldRetry: true,+ delayMs: 400,+ tracker: {+ attempts: 3,+ latestSequence: 3,+ highestObservedSequence: 5,+ },+ });+ expect(fourth).toEqual({+ shouldRetry: false,+ delayMs: 0,+ tracker: null,+ });+ });++ it("resets the retry budget when the replay frontier changes", () => {+ const exhausted = {+ attempts: 3,+ latestSequence: 3,+ highestObservedSequence: 5,+ };++ expect(+ deriveReplayRetryDecision({+ previousTracker: exhausted,+ completion: {+ replayMadeProgress: false,+ shouldReplay: true,+ },+ recoveryState: {+ latestSequence: 3,+ highestObservedSequence: 6,+ },+ baseDelayMs: 100,+ maxNoProgressRetries: 3,+ }),+ ).toEqual({+ shouldRetry: true,+ delayMs: 100,+ tracker: {+ attempts: 1,+ latestSequence: 3,+ highestObservedSequence: 6,+ },+ });+ });+});diff --git a/apps/web/src/orchestrationRecovery.ts b/apps/web/src/orchestrationRecovery.tsindex 5af48f85b..270461bd2 100644--- a/apps/web/src/orchestrationRecovery.ts+++ b/apps/web/src/orchestrationRecovery.ts@@ -13,8 +13,74 @@ export interface OrchestrationRecoveryState {inFlight: OrchestrationRecoveryPhase | null;}+export interface ReplayRecoveryCompletion {+ replayMadeProgress: boolean;+ shouldReplay: boolean;+}++export interface ReplayRetryTracker {+ attempts: number;+ latestSequence: number;+ highestObservedSequence: number;+}++export interface ReplayRetryDecision {+ shouldRetry: boolean;+ delayMs: number;+ tracker: ReplayRetryTracker | null;+}+type SequencedEvent = Readonly<{ sequence: number }>;+export function deriveReplayRetryDecision(input: {+ previousTracker: ReplayRetryTracker | null;+ completion: ReplayRecoveryCompletion;+ recoveryState: Pick<OrchestrationRecoveryState, "latestSequence" | "highestObservedSequence">;+ baseDelayMs: number;+ maxNoProgressRetries: number;+}): ReplayRetryDecision {+ if (!input.completion.shouldReplay) {+ return {+ shouldRetry: false,+ delayMs: 0,+ tracker: null,+ };+ }++ if (input.completion.replayMadeProgress) {+ return {+ shouldRetry: true,+ delayMs: 0,+ tracker: null,+ };+ }++ const previousTracker = input.previousTracker;+ const sameFrontier =+ previousTracker !== null &&+ previousTracker.latestSequence === input.recoveryState.latestSequence &&+ previousTracker.highestObservedSequence === input.recoveryState.highestObservedSequence;++ const attempts = sameFrontier && previousTracker !== null ? previousTracker.attempts + 1 : 1;+ if (attempts > input.maxNoProgressRetries) {+ return {+ shouldRetry: false,+ delayMs: 0,+ tracker: null,+ };+ }++ return {+ shouldRetry: true,+ delayMs: input.baseDelayMs * 2 ** (attempts - 1),+ tracker: {+ attempts,+ latestSequence: input.recoveryState.latestSequence,+ highestObservedSequence: input.recoveryState.highestObservedSequence,+ },+ };+}+export function createOrchestrationRecoveryCoordinator() {let state: OrchestrationRecoveryState = {latestSequence: 0,@@ -120,16 +186,16 @@ export function createOrchestrationRecoveryCoordinator() {return true;},- completeReplayRecovery(): boolean {+ completeReplayRecovery(): ReplayRecoveryCompletion {const replayMadeProgress =replayStartSequence !== null && state.latestSequence > replayStartSequence;replayStartSequence = null;state.inFlight = null;- if (!replayMadeProgress) {- state.pendingReplay = false;- return false;- }- return resolveReplayNeedAfterRecovery().shouldReplay;+ const replayResolution = resolveReplayNeedAfterRecovery();+ return {+ replayMadeProgress,+ shouldReplay: replayResolution.shouldReplay,+ };},failReplayRecovery(): void {diff --git a/apps/web/src/routes/__root.tsx b/apps/web/src/routes/__root.tsxindex d6f56e531..a750f6fd2 100644--- a/apps/web/src/routes/__root.tsx+++ b/apps/web/src/routes/__root.tsx@@ -42,6 +42,7 @@ import { projectQueryKeys } from "../lib/projectReactQuery";import { collectActiveTerminalThreadIds } from "../lib/terminalStateCleanup";import { deriveOrchestrationBatchEffects } from "../orchestrationEventEffects";import { createOrchestrationRecoveryCoordinator } from "../orchestrationRecovery";+import { deriveReplayRetryDecision } from "../orchestrationRecovery";import { getWsRpcClient } from "~/wsRpcClient";export const Route = createRootRouteWithContext<{@@ -188,6 +189,9 @@ function coalesceOrchestrationUiEvents(return coalesced;}+const REPLAY_RECOVERY_RETRY_DELAY_MS = 100;+const MAX_NO_PROGRESS_REPLAY_RETRIES = 3;+function ServerStateBootstrap() {useEffect(() => startServerStateSync(getWsRpcClient().server), []);@@ -309,6 +313,7 @@ function EventRouter() {let disposed = false;disposedRef.current = false;const recovery = createOrchestrationRecoveryCoordinator();+ let replayRetryTracker: import("../orchestrationRecovery").ReplayRetryTracker | null = null;let needsProviderInvalidation = false;const pendingDomainEvents: OrchestrationEvent[] = [];let flushPendingDomainEventsScheduled = false;@@ -435,13 +440,42 @@ function EventRouter() {applyEventBatch(events);}} catch {+ replayRetryTracker = null;recovery.failReplayRecovery();void fallbackToSnapshotRecovery();return;}- if (!disposed && recovery.completeReplayRecovery()) {- void recoverFromSequenceGap();+ if (!disposed) {+ const replayCompletion = recovery.completeReplayRecovery();+ const retryDecision = deriveReplayRetryDecision({+ previousTracker: replayRetryTracker,+ completion: replayCompletion,+ recoveryState: recovery.getState(),+ baseDelayMs: REPLAY_RECOVERY_RETRY_DELAY_MS,+ maxNoProgressRetries: MAX_NO_PROGRESS_REPLAY_RETRIES,+ });+ replayRetryTracker = retryDecision.tracker;++ if (retryDecision.shouldRetry) {+ if (retryDecision.delayMs > 0) {+ await new Promise<void>((resolve) => {+ setTimeout(resolve, retryDecision.delayMs);+ });+ if (disposed) {+ return;+ }+ }+ void recoverFromSequenceGap();+ } else if (replayCompletion.shouldReplay && import.meta.env.MODE !== "test") {+ console.warn(+ "[orchestration-recovery]",+ "Stopping replay recovery after no-progress retries.",+ {+ state: recovery.getState(),+ },+ );+ }}};