Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
refactor(cache): run the mutation queue as an explicit state machine (#7824)
teo23a35cac590d
diff --git a/apps/web/src/lib/graphql-cache/exchange/mutation-queue-runner.ts b/apps/web/src/lib/graphql-cache/exchange/mutation-queue-runner.tsnew file mode 100644index 0000000000..ffb4ae0c34--- /dev/null+++ b/apps/web/src/lib/graphql-cache/exchange/mutation-queue-runner.ts@@ -0,0 +1,948 @@+/**+ * Strict runner for the durable optimistic mutation queue.+ *+ * The cache worker owns the queue; this runner claims its head, decides how+ * to send it, and settles the outcome. Two pieces of state drive it:+ *+ * - The pump (`drainQueue`) runs on a timer, `online`, or cache restore and+ * claims the next runnable head while no attempt is active. A durable+ * enqueue may also claim its own head immediately (`admit`).+ * - The head machine tracks the claimed attempt:+ *+ * idle | confirmed-unsettled+ * │ claim+ * ├─ same head as confirmed ──▶ settling (re-apply; never resend)+ * ▼+ * preparing ── superseded discard ──▶ idle+ * │ └─ veto or recovery failure ──▶ settling+ * ▼+ * in-flight ── replay rejected unsettled ──▶ settling+ * ▼+ * settling ──▶ idle, or confirmed-unsettled when the server+ * confirmed the head but its local commit failed+ *+ * Every attempt carries a token: the storage generation it was claimed in plus+ * its lease owner and generation. The lease fences durable writes in the+ * worker; the storage generation fences this runner. A storage reset bumps+ * the generation and returns the head to idle, which retires every token.+ * Head changes go through `transition`, which checks the token, and each+ * awaited step rechecks `isCurrent` before it acts. Releasing live callers as+ * queued and scheduling the pump are idempotent wakeups and are not fenced.+ */++import {+ type Client,+ CombinedError,+ createRequest,+ makeOperation,+ type Operation,+ type OperationResult,+} from '@urql/core';+import { type DocumentNode, Kind, parse } from 'graphql';+import { match } from 'ts-pattern';+import type { CacheHost } from '../host/types';+import type {+ ClaimedMutation,+ EnqueueOptimisticMutationResult,+ MutationClaim,+ QueryRevalidationWire,+} from '../protocol';+import type { NormalizedCacheExchangeOptions } from './normalized-cache-exchange';+import {+ notifyOptimisticMutationEnqueued,+ optimisticContextOf,+ withOptimisticMutationDisposition,+} from './optimistic';++/**+ * Private operation-context field carrying the optimistic transaction id+ * from forward time to result time. Lives on the operation (not keyed by+ * urql operation key): identical concurrent mutations share a key but each+ * carries its own transaction.+ */+const QUEUE_ATTEMPT_CONTEXT_KEY = 'normalizedCacheQueueAttempt';+export const QUEUE_REQUEST_TIMEOUT_MS = 60_000;+export const QUEUE_LEASE_MS = 5 * 60_000;+const EMPTY_QUEUE_POLL_MS = 30_000;+const MAX_MUTATION_SERVER_FAILURES = 10;+const MUTATION_LIFECYCLE_TIMEOUT_MS = 2_000;++async function boundedMutationLifecycle<T>(work: () => Promise<T>): Promise<T> {+ let timer: ReturnType<typeof setTimeout> | undefined;+ try {+ return await Promise.race([+ work(),+ new Promise<never>((_, reject) => {+ timer = setTimeout(+ () => reject(new Error('Mutation recovery timed out')),+ MUTATION_LIFECYCLE_TIMEOUT_MS+ );+ }),+ ]);+ } finally {+ if (timer !== undefined) clearTimeout(timer);+ }+}++/** Storage generation captured when work starts; a reset retires it. */+export type StorageEpoch = { storageGeneration: number };++/** Fences one claimed attempt in this runner and in the durable queue. */+type AttemptToken = StorageEpoch & {+ transactionId: string;+ leaseOwner: string;+ leaseGeneration: string;+};++export type QueueAttemptContext = AttemptToken & {+ mutation?: ClaimedMutation;+ attemptCount: number;+ serverFailureCount: number;+ /** Transport-only metadata; urql drops the response on error-free payloads. */+ response?: Response;+};++function claimOf(token: AttemptToken): MutationClaim {+ return { owner: token.leaseOwner, generation: token.leaseGeneration };+}++export function queueAttemptOf(op: Operation): QueueAttemptContext | undefined {+ const value: unknown = op.context[QUEUE_ATTEMPT_CONTEXT_KEY];+ if (+ value !== null &&+ typeof value === 'object' &&+ 'transactionId' in value &&+ 'leaseOwner' in value &&+ 'leaseGeneration' in value+ ) {+ return value as QueueAttemptContext;+ }+ return undefined;+}++function withQueueAttempt(+ op: Operation,+ attempt: QueueAttemptContext+): Operation {+ return makeOperation(op.kind, op, {+ ...op.context,+ [QUEUE_ATTEMPT_CONTEXT_KEY]: attempt,+ });+}++export function replayDocument(query: string, name?: string): DocumentNode {+ const document = parse(query);+ if (!name) return document;+ const definitions = document.definitions.filter(+ (definition) =>+ definition.kind !== Kind.OPERATION_DEFINITION ||+ definition.name?.value === name+ );+ if (+ !definitions.some(+ (definition) => definition.kind === Kind.OPERATION_DEFINITION+ )+ ) {+ throw new Error(`queued GraphQL operation ${name} is missing`);+ }+ return { ...document, definitions };+}++function retryDelayMs(attemptCount: number): number {+ return Math.min(1_000 * 2 ** Math.max(0, attemptCount - 1), 60_000);+}++/** Bounds network time inside the lease and retains HTTP status for settlement. */+export function withQueueRequestTimeout(op: Operation): Operation {+ const operationFetch = op.context.fetch ?? globalThis.fetch;+ const attempt = queueAttemptOf(op);+ return makeOperation(op.kind, op, {+ ...op.context,+ fetch: async (input, init) => {+ const timeoutSignal = AbortSignal.timeout(QUEUE_REQUEST_TIMEOUT_MS);+ const response = await operationFetch(input, {+ ...init,+ signal: init?.signal+ ? AbortSignal.any([init.signal, timeoutSignal])+ : timeoutSignal,+ });+ if (attempt) attempt.response = response;+ return response;+ },+ });+}++function queuedMutationResult(+ op: Operation,+ transactionId: string+): OperationResult {+ return withOptimisticMutationDisposition(+ {+ operation: op,+ data: optimisticContextOf(op)?.optimisticResponse,+ error: undefined,+ extensions: undefined,+ stale: false,+ hasNext: false,+ },+ { kind: 'queued', transactionId }+ );+}++/** Only an authoritative GraphQL rejection carries a domain failure code. */+function mutationErrorCode(+ error: CombinedError | undefined+): string | undefined {+ if (error?.networkError) return;+ const code = error?.graphQLErrors[0]?.extensions.code;+ return typeof code === 'string' ? code : undefined;+}++/** Server response kept until its local commit succeeds. */+type ConfirmedMutation = { transactionId: string; result: OperationResult };++/** Phase of the claimed queue head; at most one attempt is active. */+type HeadState =+ | { kind: 'idle' }+ /** The server confirmed this head but its local commit failed. */+ | { kind: 'confirmed-unsettled'; confirmed: ConfirmedMutation }+ /** Startup recovery and `beforeMutationAttempt` admission. */+ | { kind: 'preparing' }+ /** Forwarded by its live caller or replayed from durable data. */+ | { kind: 'in-flight' }+ /** Writing the attempt's durable outcome. */+ | { kind: 'settling'; confirmed?: ConfirmedMutation };++const IDLE: HeadState = { kind: 'idle' };+const PREPARING: HeadState = { kind: 'preparing' };+const IN_FLIGHT: HeadState = { kind: 'in-flight' };+const SETTLING: HeadState = { kind: 'settling' };++/** An active attempt holds the head; the pump must not claim another. */+function isActive(head: HeadState): boolean {+ return match(head.kind)+ .with('preparing', 'in-flight', 'settling', () => true)+ .with('idle', 'confirmed-unsettled', () => false)+ .exhaustive();+}++function confirmedOf(head: HeadState): ConfirmedMutation | undefined {+ return head.kind === 'confirmed-unsettled' || head.kind === 'settling'+ ? head.confirmed+ : undefined;+}++/** A network result for the in-flight head starts its settlement. */+function receiveResult(head: HeadState): HeadState {+ return head.kind === 'in-flight' ? SETTLING : head;+}++function confirm(confirmed: ConfirmedMutation) {+ return (head: HeadState): HeadState =>+ isActive(head)+ ? { kind: 'settling', confirmed }+ : { kind: 'confirmed-unsettled', confirmed };+}++function forgetConfirmed(head: HeadState): HeadState {+ if (head.kind === 'confirmed-unsettled') return IDLE;+ return head.kind === 'settling' ? SETTLING : head;+}++/** Releases the head; an unapplied server confirmation stays pinned. */+function release(head: HeadState): HeadState {+ const confirmed = confirmedOf(head);+ return confirmed ? { kind: 'confirmed-unsettled', confirmed } : IDLE;+}++type LiveQueuedOperation = {+ operation: Operation;+ resolveRoute: (result: OperationResult | undefined) => void;+};++export type MutationQueueRunnerDeps = {+ host: CacheHost;+ client: Client;+ options: Pick<+ NormalizedCacheExchangeOptions,+ | 'prepareMutationQueue'+ | 'beforeMutationAttempt'+ | 'onMutationAttemptResult'+ | 'shouldRetryMutation'+ | 'onCacheError'+ >;+ /** Lease owner for this exchange's claims. */+ owner: string;+ /** Sends an operation down the exchange's network stream. */+ forward: (op: Operation) => void;+ /** Delivers a result through the exchange's normal write path. */+ writeThrough: (+ result: OperationResult+ ) => Promise<OperationResult | undefined>;+ /** Query text and variables the cache normalizes `op`'s data against. */+ cacheDocument: (op: Operation) => {+ query: string;+ operationName?: string;+ variables?: Record<string, unknown>;+ };+ /** Evicts records a successful response implies were deleted. */+ deleteReportedRecords: (result: OperationResult) => Promise<unknown>;+ /**+ * Replays explicit deletions mixed into a committed response, in order.+ * Undefined when there is nothing to replay, so settlement does not yield.+ */+ replayExplicitEffects: (+ op: Operation,+ data: unknown,+ isCurrent: () => boolean+ ) => Promise<unknown> | undefined;+ /** Refreshes queries a settled mutation could not update in place. */+ revalidate: (+ revalidations: QueryRevalidationWire[],+ mutation?: Operation+ ) => Promise<void>;+};++/** The exchange's view of the queue runner. */+export type MutationQueueRunner = {+ /** Captures the storage epoch an enqueue starts in. */+ epoch: () => StorageEpoch;+ /** False once a storage reset has retired `epoch`. */+ isCurrent: (epoch: StorageEpoch) => boolean;+ /** Holds a durably enqueued caller until the strict queue routes it. */+ admit: (+ op: Operation,+ enqueue: EnqueueOptimisticMutationResult,+ epoch: StorageEpoch+ ) => Promise<OperationResult | undefined>;+ /** Settles a queue attempt's network result. */+ settle: (+ result: OperationResult,+ attempt: QueueAttemptContext+ ) => Promise<OperationResult>;+ /** Schedules the first drain. */+ start: () => void;+ /** Probes the queue now, keeping the request until a claim can run. */+ wake: () => void;+ /** Retires every attempt token and returns the head to idle. */+ resetStorage: () => void;+};++export function createMutationQueueRunner(+ deps: MutationQueueRunnerDeps+): MutationQueueRunner {+ const { host, client, options, owner, forward, writeThrough } = deps;+ const liveQueuedOps = new Map<string, LiveQueuedOperation>();+ let storageGeneration = 0;+ // The durable queue is strictly ordered. A confirmed head keeps its server+ // response until local settlement succeeds, without repeating a server+ // effect after a cache failure. Process loss still requires server-side+ // idempotency to make replay safe across runners.+ let head: HeadState = IDLE;+ // Pump state: wakeups and the retry hint are independent of the head.+ let queuePreparation: Promise<void> | undefined;+ let drainRunning = false;+ let drainRequested = false;+ let deferredUntil: number | undefined;+ let drainTimer: ReturnType<typeof setTimeout> | undefined;++ function isCurrent(token: StorageEpoch): boolean {+ return token.storageGeneration === storageGeneration;+ }++ /** Moves the head only while `token` has survived every storage reset. */+ function transition(+ token: StorageEpoch,+ next: HeadState | ((head: HeadState) => HeadState)+ ): boolean {+ if (!isCurrent(token)) return false;+ head = typeof next === 'function' ? next(head) : next;+ return true;+ }++ function scheduleDrain(delayMs = 0): void {+ if (drainTimer !== undefined) clearTimeout(drainTimer);+ drainTimer = setTimeout(+ () => {+ drainTimer = undefined;+ void drainQueue();+ },+ drainRequested ? 0 : delayMs+ );+ }++ function wakeDrain(): void {+ // Keep the wakeup until the runner can claim again. An interrupted+ // claim/settlement may still be unwinding and scheduling its backoff.+ drainRequested = true;+ scheduleDrain();+ }++ function resolveLiveOperationsAsQueued(): void {+ for (const [transactionId, live] of liveQueuedOps) {+ liveQueuedOps.delete(transactionId);+ live.resolveRoute(queuedMutationResult(live.operation, transactionId));+ }+ }++ /**+ * Routes one leased queue head to local settlement or the network. Await+ * points match the pre-extraction runner: forwarding a live caller must+ * stay synchronous with its claim, or callers behind it would be released.+ */+ async function routeClaimedMutation(+ claimed: ClaimedMutation,+ epoch: StorageEpoch+ ): Promise<void> {+ const attempt: QueueAttemptContext = {+ mutation: claimed,+ transactionId: claimed.transactionId,+ leaseOwner: owner,+ leaseGeneration: claimed.leaseGeneration,+ attemptCount: claimed.attemptCount,+ serverFailureCount: claimed.serverFailureCount ?? 0,+ storageGeneration: epoch.storageGeneration,+ };+ const confirmed = confirmedOf(head);+ if (confirmed?.transactionId === claimed.transactionId) {+ // claim → settling: re-apply the server's response before admission+ // hooks run, so they can never veto or resend a confirmed head.+ if (!transition(attempt, { kind: 'settling', confirmed })) return;+ deferredUntil = undefined;+ await writeThrough({+ ...confirmed.result,+ operation: withQueueAttempt(confirmed.result.operation, attempt),+ });+ resolveLiveOperationsAsQueued();+ return;+ }+ // claim → preparing. Claiming another head proves the previous one was+ // removed, including settlement by another runner or a storage reset.+ if (!transition(attempt, PREPARING)) return;+ try {+ if (options.prepareMutationQueue) await prepareQueueRecovery();+ if (+ options.beforeMutationAttempt &&+ !(await boundedMutationLifecycle(() =>+ options.beforeMutationAttempt!(claimed)+ ))+ ) {+ // preparing → settling → idle: discard obsolete local intent.+ if (!transition(attempt, SETTLING)) return;+ await host.rollbackOptimisticWrite(+ claimed.transactionId,+ claimOf(attempt),+ 'Obsolete local intent',+ 'LOCAL_SUPERSEDED'+ );+ if (!transition(attempt, IDLE)) return;+ resolveLiveOperationsAsQueued();+ scheduleDrain();+ return;+ }+ } catch {+ await failPreparation(claimed, attempt);+ return;+ }+ // A superseded create can already exist on the server. Replay it to+ // recover its identity before sending the newer edit or discard.+ // Older hosts omit this flag: conservatively replay rather than loop+ // forever asking the engine to discard a write it must retain.+ const discard =+ claimed.superseded && claimed.requiresConfirmation === false;+ // preparing → idle (discard) or in-flight (send).+ if (!transition(attempt, discard ? IDLE : IN_FLIGHT)) return;+ deferredUntil = undefined;+ if (discard) {+ const discarded = await host.deferOptimisticWrite(+ claimed.transactionId,+ claimOf(attempt),+ Date.now(),+ 'superseded before network replay'+ );+ if (discarded.kind !== 'discarded-superseded') {+ throw new Error('superseded mutation was unexpectedly deferred');+ }+ const live = liveQueuedOps.get(claimed.transactionId);+ if (live) {+ liveQueuedOps.delete(claimed.transactionId);+ live.resolveRoute(+ queuedMutationResult(live.operation, claimed.transactionId)+ );+ }+ resolveLiveOperationsAsQueued();+ scheduleDrain();+ return;+ }+ const live = liveQueuedOps.get(claimed.transactionId);+ if (live) {+ liveQueuedOps.delete(claimed.transactionId);+ live.resolveRoute(undefined);+ forward(+ withQueueRequestTimeout(withQueueAttempt(live.operation, attempt))+ );+ } else {+ let replayOperation: Operation | undefined;+ try {+ const replay = client.mutation(+ replayDocument(claimed.query, claimed.operationName),+ claimed.variables,+ {+ requestPolicy: 'network-only',+ [QUEUE_ATTEMPT_CONTEXT_KEY]: attempt,+ }+ );+ await replay.toPromise().then((result) => {+ replayOperation = result.operation;+ // Normal exchange results settle the attempt in writeThrough+ // before resolving. Reject only an otherwise-unhandled error.+ if (result.error && isActive(head)) {+ return Promise.reject(result.error);+ }+ });+ } catch (error) {+ try {+ // in-flight → settling → idle: the replay never reached settlement.+ if (!transition(attempt, SETTLING)) return;+ const rolledBack = await host.rollbackOptimisticWrite(+ claimed.transactionId,+ claimOf(attempt),+ error instanceof Error ? error.message : String(error),+ error instanceof CombinedError+ ? mutationErrorCode(error)+ : undefined+ );+ if (!isCurrent(attempt)) return;+ if (rolledBack.kind === 'rolled-back') {+ void deps.revalidate(+ rolledBack.revalidations ?? [],+ replayOperation+ );+ }+ } finally {+ if (transition(attempt, IDLE)) scheduleDrain();+ }+ }+ }+ // Any other live operation is ordered behind the claimed head.+ resolveLiveOperationsAsQueued();+ }++ /** preparing → settling → idle for a head whose recovery state failed. */+ async function failPreparation(+ claimed: ClaimedMutation,+ attempt: QueueAttemptContext+ ): Promise<void> {+ queuePreparation = undefined;+ try {+ if (!transition(attempt, SETTLING)) return;+ const live = liveQueuedOps.get(claimed.transactionId);+ const operation =+ live?.operation ??+ makeOperation(+ 'mutation',+ createRequest(+ replayDocument(claimed.query, claimed.operationName),+ claimed.variables+ ),+ { url: '', requestPolicy: 'network-only' }+ );+ const failure = new CombinedError({+ graphQLErrors: [+ {+ message: 'Unable to prepare the saved mutation for replay',+ extensions: { code: 'LOCAL_RECOVERY_FAILED' },+ },+ ],+ });+ const result: OperationResult = {+ operation,+ data: undefined,+ error: failure,+ stale: false,+ hasNext: false,+ };+ await recordAttemptResult(attempt, result, false);+ // Known gap: unlike every other settlement step, this rollback and the+ // bookkeeping after it do not recheck the token. A storage reset during+ // the recovery hook lets this stale attempt roll back a replacement+ // queue head that reuses its transaction id and lease generation.+ await host.rollbackOptimisticWrite(+ claimed.transactionId,+ claimOf(attempt),+ failure.message,+ 'LOCAL_RECOVERY_FAILED'+ );+ liveQueuedOps.delete(claimed.transactionId);+ live?.resolveRoute(+ withOptimisticMutationDisposition(result, {+ kind: 'permanently-failed',+ transactionId: claimed.transactionId,+ })+ );+ deferredUntil = undefined;+ } finally {+ if (transition(attempt, IDLE)) {+ resolveLiveOperationsAsQueued();+ scheduleDrain();+ }+ }+ }++ async function drainQueue(): Promise<void> {+ if (isActive(head)) {+ // Every newly enqueued operation is behind the claimed head. Its+ // caller can stop waiting; durable replay now owns the mutation.+ resolveLiveOperationsAsQueued();+ return;+ }+ if (drainRunning) return;+ drainRunning = true;+ drainRequested = false;+ try {+ // Restore can invalidate the generation while the host initializes.+ // Finish that handshake before tagging a claim or starting its lease.+ await host.currentRevision();+ if (options.prepareMutationQueue) await prepareQueueRecovery();+ const now = Date.now();+ // A wakeup probes immediately, but must retain a future retry+ // deadline if the durable head is not eligible yet. Consume expired+ // hints so a head leased by another runner cannot cause a busy loop.+ if (deferredUntil !== undefined && deferredUntil <= now) {+ deferredUntil = undefined;+ }+ const epoch: StorageEpoch = { storageGeneration };+ const claimed = await host.claimNextMutation(+ owner,+ now,+ now + QUEUE_LEASE_MS+ );+ if (!claimed) {+ resolveLiveOperationsAsQueued();+ // Short retries keep their deadline; uncertain five-minute lease+ // backoffs must not prevent regular polling after a wakeup.+ scheduleDrain(+ deferredUntil === undefined+ ? EMPTY_QUEUE_POLL_MS+ : Math.min(+ EMPTY_QUEUE_POLL_MS,+ Math.max(0, deferredUntil - Date.now())+ )+ );+ return;+ }++ await routeClaimedMutation(claimed, epoch);+ } catch {+ // Enqueue already succeeded, so callers must observe these as+ // queued even if the runner cannot currently inspect the head.+ resolveLiveOperationsAsQueued();+ scheduleDrain(EMPTY_QUEUE_POLL_MS);+ } finally {+ drainRunning = false;+ if (drainRequested) scheduleDrain();+ }+ }++ async function prepareQueueRecovery(): Promise<void> {+ try {+ queuePreparation ??= boundedMutationLifecycle(() =>+ options.prepareMutationQueue!()+ );+ await queuePreparation;+ } catch (error) {+ queuePreparation = undefined;+ console.warn(+ '[graphql-cache] Unable to prepare mutation recovery',+ error+ );+ }+ }++ async function recordAttemptResult(+ attempt: QueueAttemptContext,+ result: OperationResult,+ retry: boolean+ ): Promise<void> {+ if (!attempt.mutation || !options.onMutationAttemptResult) return;+ try {+ await boundedMutationLifecycle(() =>+ options.onMutationAttemptResult!(attempt.mutation!, result, retry)+ );+ } catch (error) {+ try {+ options.onCacheError?.(error, result.operation);+ } catch {+ /* Diagnostics cannot retain a failed queue head. */+ }+ }+ }++ async function admit(+ op: Operation,+ enqueue: EnqueueOptimisticMutationResult,+ epoch: StorageEpoch+ ): Promise<OperationResult | undefined> {+ if (enqueue.upsertKind.kind === 'replaced-pending') {+ const superseded = liveQueuedOps.get(+ enqueue.upsertKind.removedTransactionId+ );+ if (superseded) {+ liveQueuedOps.delete(enqueue.upsertKind.removedTransactionId);+ superseded.resolveRoute(+ queuedMutationResult(+ superseded.operation,+ enqueue.upsertKind.removedTransactionId+ )+ );+ }+ }+ const routed = new Promise<OperationResult | undefined>((resolve) => {+ liveQueuedOps.set(enqueue.transactionId, {+ operation: op,+ resolveRoute: resolve,+ });+ });+ notifyOptimisticMutationEnqueued(op);+ try {+ await match(enqueue.initialClaim)+ .with({ kind: 'claimed' }, ({ mutation }) =>+ routeClaimedMutation(mutation, epoch)+ )+ .with({ kind: 'not-runnable' }, () => {+ resolveLiveOperationsAsQueued();+ scheduleDrain();+ })+ .with({ kind: 'failed' }, ({ error }) => {+ options.onCacheError?.(new Error(error), op);+ resolveLiveOperationsAsQueued();+ scheduleDrain(EMPTY_QUEUE_POLL_MS);+ })+ .exhaustive();+ } catch (error) {+ // Enqueue already succeeded. Preserve durable-runner ownership even+ // if routing or claim rollback fails unexpectedly.+ options.onCacheError?.(error, op);+ resolveLiveOperationsAsQueued();+ scheduleDrain(EMPTY_QUEUE_POLL_MS);+ }+ return await routed;+ }++ async function settle(+ networkResult: OperationResult,+ attempt: QueueAttemptContext+ ): Promise<OperationResult> {+ let result = networkResult;+ const op = result.operation;+ const detachedResult = () =>+ withOptimisticMutationDisposition(result, {+ kind:+ result.data != null && !result.error+ ? 'committed'+ : 'permanently-failed',+ transactionId: attempt.transactionId,+ });+ if (!transition(attempt, receiveResult)) return detachedResult();+ const claim = claimOf(attempt);+ let retryAt: number | undefined;+ let settled = false;+ let replacementTransactionId: string | undefined;+ let disposition:+ | 'committed'+ | 'queued'+ | 'superseded'+ | 'permanently-failed' = 'queued';+ try {+ const response = result.error?.response ?? attempt.response;+ const httpServerFailure = (response?.status ?? 0) >= 500;+ // A valid GraphQL payload can arrive over HTTP 5xx without an+ // urql error. Never commit it or bypass the retry policy.+ if (httpServerFailure && !result.error) {+ result = {+ ...result,+ data: undefined,+ error: new CombinedError({+ networkError: new Error(`HTTP ${response.status}`),+ response,+ }),+ };+ }+ if (result.error || result.data == null) {+ let retry = false;+ if (result.error && options.shouldRetryMutation) {+ try {+ retry = await options.shouldRetryMutation(result.error);+ } catch (error) {+ options.onCacheError?.(error, op);+ }+ }+ if (!isCurrent(attempt)) return detachedResult();+ // urql represents HTTP 5xx as networkError too. Count actual+ // server responses, never connection failures or timeouts.+ const serverFailure =+ httpServerFailure || (result.error?.graphQLErrors.length ?? 0) > 0;+ if (+ retry &&+ serverFailure &&+ attempt.serverFailureCount + 1 >= MAX_MUTATION_SERVER_FAILURES+ ) {+ retry = false;+ result = {+ ...result,+ error: new CombinedError({+ graphQLErrors: [+ {+ message: `Mutation stopped after ${MAX_MUTATION_SERVER_FAILURES} server failures: ${result.error?.message}`,+ extensions: { code: 'MUTATION_RETRY_EXHAUSTED' },+ },+ ],+ response: result.error?.response,+ }),+ };+ }+ await recordAttemptResult(attempt, result, retry);+ if (!isCurrent(attempt)) return detachedResult();+ if (retry) {+ retryAt = Date.now() + retryDelayMs(attempt.attemptCount);+ const deferred = await host.deferOptimisticWrite(+ attempt.transactionId,+ claim,+ retryAt,+ result.error?.message ?? 'mutation returned no data',+ serverFailure+ );+ if (!isCurrent(attempt)) return detachedResult();+ settled = true;+ if (deferred.kind === 'discarded-superseded') {+ retryAt = undefined;+ replacementTransactionId = deferred.replacementTransactionId;+ disposition = 'superseded';+ } else {+ disposition = 'queued';+ }+ } else {+ const rolledBack = await host.rollbackOptimisticWrite(+ attempt.transactionId,+ claim,+ result.error?.message ?? 'mutation returned no data',+ mutationErrorCode(result.error)+ );+ if (!isCurrent(attempt)) return detachedResult();+ settled = true;+ if (rolledBack.kind === 'discarded-superseded') {+ replacementTransactionId = rolledBack.replacementTransactionId;+ disposition = 'superseded';+ } else {+ disposition = 'permanently-failed';+ void deps.revalidate(rolledBack.revalidations ?? [], op);+ }+ }+ } else {+ const confirmed = { transactionId: attempt.transactionId, result };+ if (!transition(attempt, confirm(confirmed))) return detachedResult();+ await recordAttemptResult(attempt, result, false);+ if (!isCurrent(attempt)) return detachedResult();+ const committed = await host.commitOptimisticWrite(+ attempt.transactionId,+ claim,+ { ...deps.cacheDocument(op), data: result.data }+ );+ if (!transition(attempt, forgetConfirmed)) return detachedResult();+ settled = true;+ for (const error of committed.identityErrors ?? []) {+ try {+ options.onCacheError?.(new Error(error), op);+ } catch {+ // A diagnostic callback cannot undo confirmed settlement.+ }+ }+ if (committed.kind === 'failed') {+ const error = new Error(committed.error);+ try {+ options.onCacheError?.(error, op);+ } catch {+ // A diagnostic callback cannot undo confirmed settlement.+ }+ result = {+ ...result,+ data: undefined,+ error: new CombinedError({ networkError: error }),+ };+ replacementTransactionId = committed.replacementTransactionId;+ disposition = replacementTransactionId+ ? 'superseded'+ : 'permanently-failed';+ if (!replacementTransactionId) {+ void deps.revalidate(committed.revalidations ?? [], op);+ }+ } else if (committed.kind === 'committed-superseded') {+ replacementTransactionId = committed.replacementTransactionId;+ disposition = 'superseded';+ await deps.deleteReportedRecords(result);+ } else {+ disposition = 'committed';+ await deps.deleteReportedRecords(result);+ if (!isCurrent(attempt)) return detachedResult();+ const replay = deps.replayExplicitEffects(op, result.data, () =>+ isCurrent(attempt)+ );+ if (replay) await replay;+ if (isCurrent(attempt))+ await deps.revalidate(committed.revalidations ?? [], op);+ }+ }+ } catch (error) {+ try {+ options.onCacheError?.(error, op);+ } catch {+ // Reporting cannot discard the retained response or stop retries.+ }+ if (!settled && isCurrent(attempt)) {+ retryAt = Date.now() + QUEUE_LEASE_MS;+ // A lost acknowledgement may hide a completed transaction.+ // Preserve optimism and let a fresh durable claim decide.+ disposition = 'queued';+ }+ } finally {+ if (transition(attempt, release)) {+ liveQueuedOps.delete(attempt.transactionId);+ deferredUntil = retryAt;+ scheduleDrain(+ retryAt === undefined ? 0 : Math.max(0, retryAt - Date.now())+ );+ }+ }+ return withOptimisticMutationDisposition(+ result,+ disposition === 'superseded'+ ? {+ kind: 'superseded',+ transactionId: attempt.transactionId,+ replacementTransactionId:+ replacementTransactionId ?? attempt.transactionId,+ }+ : {+ kind: disposition,+ transactionId: attempt.transactionId,+ }+ );+ }++ function resetStorage(): void {+ storageGeneration += 1;+ head = IDLE;+ deferredUntil = undefined;+ resolveLiveOperationsAsQueued();+ }++ return {+ epoch: () => ({ storageGeneration }),+ isCurrent,+ admit,+ settle,+ start: () => scheduleDrain(),+ wake: wakeDrain,+ resetStorage,+ };+}diff --git a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.tsindex 13e6674a7b..06323aa59a 100644--- a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts+++ b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts@@ -3712,6 +3712,106 @@ describe('normalizedCacheExchange', () => {).toEqual(['txn-1', committed ? 'restored-1' : 'restored-2']);});++ it('re-applies a confirmed head before admission hooks can discard it', async () => {+ const events: string[] = [];+ const commit = host.commitOptimisticWrite.bind(host);+ let commitFailed = false;+ vi.spyOn(host, 'commitOptimisticWrite').mockImplementation(+ async (...args) => {+ if (!commitFailed) {+ commitFailed = true;+ events.push(`commit failed ${args[0]}`);+ throw new Error('local disk unavailable');+ }+ events.push(`commit ${args[0]}`);+ return await commit(...args);+ }+ );+ const rollback = host.rollbackOptimisticWrite.bind(host);+ vi.spyOn(host, 'rollbackOptimisticWrite').mockImplementation(+ async (...args) => {+ events.push(`rollback ${args[0]} ${args[3]}`);+ return await rollback(...args);+ }+ );+ const onMutationAttemptResult = vi.fn(+ async (+ mutation: ClaimedMutation,+ _result: OperationResult,+ _retry: boolean+ ) => {+ events.push(`result ${mutation.transactionId}`);+ }+ );+ let admissions = 0;+ const { ops, forwarded } = harness(host, undefined, {+ // Only the first admission is current; later local intent is obsolete.+ beforeMutationAttempt: async (mutation) => {+ events.push(`admit ${mutation.transactionId}`);+ admissions += 1;+ return admissions === 1;+ },+ onMutationAttemptResult,+ });+ await vi.advanceTimersByTimeAsync(0);+ ops.next(makeMutationOp(1, optimistic));+ await vi.advanceTimersByTimeAsync(1);+ ops.next(makeMutationOp(2, optimistic));+ // The confirmed head is reclaimed once its lease expires.+ await vi.advanceTimersByTimeAsync(300_000);++ expect(events).toEqual([+ 'admit txn-1',+ 'result txn-1',+ 'commit failed txn-1',+ // Re-applied without sending again or asking admission to veto it.+ 'result txn-1',+ 'commit txn-1',+ 'admit txn-2',+ 'rollback txn-2 LOCAL_SUPERSEDED',+ ]);+ expect(forwarded).toHaveLength(1);+ expect(host.commits).toMatchObject([+ { transactionId: 'txn-1', data: { from: 'network' } },+ ]);+ const confirmedResult = [+ expect.objectContaining({ transactionId: 'txn-1' }),+ expect.objectContaining({ data: { from: 'network' } }),+ false,+ ];+ expect(onMutationAttemptResult.mock.calls).toEqual([+ confirmedResult,+ confirmedResult,+ ]);+ });++ // Known gap kept by the queue runner extraction: a recovery failure+ // rolls back without rechecking its attempt token. The replacement+ // queue reuses transaction id txn-1 and lease generation 1, so the+ // stale attempt discards the new head. Change to `it` once fenced.+ it.fails('fences a recovery-failure rollback after storage resets', async () => {+ const recording = deferred<void>();+ let admissions = 0;+ const { ops, forwarded } = controlledQueryHarness(host, {+ beforeMutationAttempt: async () => {+ admissions += 1;+ if (admissions === 1) throw new Error('Recovery storage lost');+ return true;+ },+ onMutationAttemptResult: () => recording.promise,+ });+ await vi.advanceTimersByTimeAsync(0);+ ops.next(makeMutationOp(1, optimistic));+ await vi.advanceTimersByTimeAsync(1);+ host.resetStorage();+ ops.next(makeMutationOp(2, optimistic));+ await vi.advanceTimersByTimeAsync(1);+ expect(forwarded).toHaveLength(1);+ recording.resolve();+ await vi.advanceTimersByTimeAsync(1);+ expect(host.rollbacks).toEqual([]);+ });});it('rolls back when a persisted replay resolves with an urql error', async () => {diff --git a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.tsindex 5058e0ff35..68d96a0036 100644--- a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts+++ b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts@@ -22,8 +22,8 @@** Mutations:* - With an optimistic response (see `executeOptimisticMutation`): the- * mutation and layer are durably queued before the ordered runner forwards- * it. Retryable failures retain optimism for background replay, but the tenth+ * mutation and layer are durably queued before the ordered runner+ * (`mutation-queue-runner.ts`) forwards it. Retryable failures retain optimism for background replay, but the tenth* server failure permanently rolls back; transport failures are not counted.* Other permanent failures roll back immediately. A caller blocked behind the queue* head receives a synthetic `queued` disposition instead of waiting.@@ -38,7 +38,6 @@import {type Client,CombinedError,- createRequest,type Exchange,makeOperation,type Operation,@@ -49,11 +48,9 @@ import {type DocumentNode,Kind,type OperationDefinitionNode,- parse,visit,} from 'graphql';import { batch } from 'solid-js';-import { match } from 'ts-pattern';import {empty,filter,@@ -87,6 +84,14 @@ import {type EntityResolverConfig,} from './entity-resolvers';import { isQueryObject, LiveQuery, withLiveQueryData } from './live-query';+import {+ createMutationQueueRunner,+ QUEUE_LEASE_MS,+ QUEUE_REQUEST_TIMEOUT_MS,+ queueAttemptOf,+ replayDocument,+ withQueueRequestTimeout,+} from './mutation-queue-runner';import type { QueryRevalidation } from './optimistic';import {normalizedEntityKey,@@ -96,13 +101,6 @@ import {} from './optimistic';import { queryShape } from './query-shape';-/**- * Private operation-context field carrying the optimistic transaction id- * from forward time to result time. Lives on the operation (not keyed by- * urql operation key): identical concurrent mutations share a key but each- * carries its own transaction.- */-const QUEUE_ATTEMPT_CONTEXT_KEY = 'normalizedCacheQueueAttempt';/** Server writes acknowledged since this query left for the network. */const QUERY_MUTATION_VERSION_CONTEXT_KEY ='normalizedCacheQueryMutationVersion';@@ -115,28 +113,6 @@ const REPLACEMENT_REGISTRATION_ONLY_CONTEXT_KEY =export const HYDRATE_ONLY_CONTEXT_KEY = 'normalizedCacheHydrateOnly';/** Retains the client-annotated document while the transport uses a stripped copy. */const HYDRATION_DOCUMENT_CONTEXT_KEY = 'normalizedCacheHydrationDocument';-const QUEUE_REQUEST_TIMEOUT_MS = 60_000;-const QUEUE_LEASE_MS = 5 * 60_000;-const EMPTY_QUEUE_POLL_MS = 30_000;-const MAX_MUTATION_SERVER_FAILURES = 10;-const MUTATION_LIFECYCLE_TIMEOUT_MS = 2_000;--async function boundedMutationLifecycle<T>(work: () => Promise<T>): Promise<T> {- let timer: ReturnType<typeof setTimeout> | undefined;- try {- return await Promise.race([- work(),- new Promise<never>((_, reject) => {- timer = setTimeout(- () => reject(new Error('Mutation recovery timed out')),- MUTATION_LIFECYCLE_TIMEOUT_MS- );- }),- ]);- } finally {- if (timer !== undefined) clearTimeout(timer);- }-}const NORMALIZED_CACHE_RESULT_METADATA_KEY = '__macroNormalizedCache';@@ -209,32 +185,6 @@ function withResultMetadata(};}-type QueueAttemptContext = {- mutation?: ClaimedMutation;- transactionId: string;- leaseOwner: string;- leaseGeneration: string;- attemptCount: number;- serverFailureCount: number;- /** Transport-only metadata; urql drops the response on error-free payloads. */- response?: Response;- storageGeneration: number;-};--function queueAttemptOf(op: Operation): QueueAttemptContext | undefined {- const value: unknown = op.context[QUEUE_ATTEMPT_CONTEXT_KEY];- if (- value !== null &&- typeof value === 'object' &&- 'transactionId' in value &&- 'leaseOwner' in value &&- 'leaseGeneration' in value- ) {- return value as QueueAttemptContext;- }- return undefined;-}-const queryTextCache = new WeakMap<object, string>();function queryText(op: Operation): string {@@ -301,48 +251,6 @@ function operationName(op: Operation): string | undefined {return undefined;}-function replayDocument(query: string, name?: string): DocumentNode {- const document = parse(query);- if (!name) return document;- const definitions = document.definitions.filter(- (definition) =>- definition.kind !== Kind.OPERATION_DEFINITION ||- definition.name?.value === name- );- if (- !definitions.some(- (definition) => definition.kind === Kind.OPERATION_DEFINITION- )- ) {- throw new Error(`queued GraphQL operation ${name} is missing`);- }- return { ...document, definitions };-}--function retryDelayMs(attemptCount: number): number {- return Math.min(1_000 * 2 ** Math.max(0, attemptCount - 1), 60_000);-}--/** Bounds network time inside the lease and retains HTTP status for settlement. */-function withQueueRequestTimeout(op: Operation): Operation {- const operationFetch = op.context.fetch ?? globalThis.fetch;- const attempt = queueAttemptOf(op);- return makeOperation(op.kind, op, {- ...op.context,- fetch: async (input, init) => {- const timeoutSignal = AbortSignal.timeout(QUEUE_REQUEST_TIMEOUT_MS);- const response = await operationFetch(input, {- ...init,- signal: init?.signal- ? AbortSignal.any([init.signal, timeoutSignal])- : timeoutSignal,- });- if (attempt) attempt.response = response;- return response;- },- });-}-function cacheResult(op: Operation,data: unknown,@@ -463,32 +371,6 @@ function uncertainEnqueueResult(op: Operation, error: Error): OperationResult {};}-function queuedMutationResult(- op: Operation,- transactionId: string-): OperationResult {- return withOptimisticMutationDisposition(- {- operation: op,- data: optimisticContextOf(op)?.optimisticResponse,- error: undefined,- extensions: undefined,- stale: false,- hasNext: false,- },- { kind: 'queued', transactionId }- );-}--/** Only an authoritative GraphQL rejection carries a domain failure code. */-function mutationErrorCode(- error: CombinedError | undefined-): string | undefined {- if (error?.networkError) return;- const code = error?.graphQLErrors[0]?.extensions.code;- return typeof code === 'string' ? code : undefined;-}-export interface NormalizedCacheExchangeOptions {/** Best-effort startup recovery before per-mutation admission. */prepareMutationQueue?: () => Promise<void>;@@ -536,7 +418,6 @@ export function normalizedCacheExchange(const activeOps = new Map<number, Operation>();const documentReader = createDocumentQueryReader(host);let cacheGeneration = 0;- let storageGeneration = 0;let mutationVersion = 0;host.onMutationSettled((settlement) => {// Another tab's durable queue runner can commit into this same cache.@@ -944,347 +825,34 @@ export function normalizedCacheExchange(};const queueOwner = `exchange:${host.clientId}`;- const liveQueuedOps = new Map<- string,- {- operation: Operation;- resolveRoute: (result: OperationResult | undefined) => void;- }- >();- // The durable queue is strictly ordered. Keep its head's confirmed- // response until local settlement succeeds, without repeating a server- // effect after a cache failure. Process loss still requires server-side- // idempotency to make replay safe across runners.- let confirmedMutation:- | { transactionId: string; result: OperationResult }- | undefined;+ const mutationQueue = createMutationQueueRunner({+ host,+ client,+ options,+ owner: queueOwner,+ forward: enqueueForward,+ writeThrough,+ cacheDocument: (op) => ({+ query: queryText(op),+ operationName: operationName(op),+ variables: op.variables as Record<string, unknown> | undefined,+ }),+ deleteReportedRecords,+ replayExplicitEffects: (op, data, isCurrent) => {+ const effects = operationCacheEffects(data);+ // Commit already normalized the complete result. Replay only mixed+ // explicit effects so their final write/delete order is identical+ // to a non-optimistic operation.+ return effects.some((effect) => effect.kind === 'delete')+ ? applyOperationCacheEffects(op, effects, isCurrent)+ : undefined;+ },+ revalidate: revalidateAfterSettlement,+ });const subscriptionEffectChains = new Map<number, Promise<boolean>>();// Teardown invalidates queued effects, including when the same operation// is immediately resubscribed after a back/forward-cache restore.const subscriptionGenerations = new Map<number, object>();- let attemptInFlight = false;- let queuePreparation: Promise<void> | undefined;- let drainRunning = false;- let drainRequested = false;- let deferredUntil: number | undefined;- let drainTimer: ReturnType<typeof setTimeout> | undefined;-- function scheduleDrain(delayMs = 0): void {- if (drainTimer !== undefined) clearTimeout(drainTimer);- drainTimer = setTimeout(- () => {- drainTimer = undefined;- void drainQueue();- },- drainRequested ? 0 : delayMs- );- }-- function wakeDrain(): void {- // Keep the wakeup until the runner can claim again. An interrupted- // claim/settlement may still be unwinding and scheduling its backoff.- drainRequested = true;- scheduleDrain();- }-- function resolveLiveOperationsAsQueued(): void {- for (const [transactionId, live] of liveQueuedOps) {- liveQueuedOps.delete(transactionId);- live.resolveRoute(- queuedMutationResult(live.operation, transactionId)- );- }- }-- /** Routes one leased queue head to local settlement or the network. */- async function routeClaimedMutation(- claimed: ClaimedMutation,- generation: number- ): Promise<void> {- if (generation !== storageGeneration) return;- const attempt: QueueAttemptContext = {- mutation: claimed,- transactionId: claimed.transactionId,- leaseOwner: queueOwner,- leaseGeneration: claimed.leaseGeneration,- attemptCount: claimed.attemptCount,- serverFailureCount: claimed.serverFailureCount ?? 0,- storageGeneration: generation,- };- if (confirmedMutation?.transactionId === claimed.transactionId) {- deferredUntil = undefined;- attemptInFlight = true;- const result = confirmedMutation.result;- await writeThrough({- ...result,- operation: makeOperation('mutation', result.operation, {- ...result.operation.context,- [QUEUE_ATTEMPT_CONTEXT_KEY]: attempt,- }),- });- resolveLiveOperationsAsQueued();- return;- }- // Claiming another head proves the previous one was removed, including- // settlement by another runner or an identity/storage reset.- confirmedMutation = undefined;- attemptInFlight = true;- try {- if (options.prepareMutationQueue) await prepareQueueRecovery();- if (- options.beforeMutationAttempt &&- !(await boundedMutationLifecycle(() =>- options.beforeMutationAttempt!(claimed)- ))- ) {- if (generation !== storageGeneration) return;- await host.rollbackOptimisticWrite(- claimed.transactionId,- { owner: queueOwner, generation: claimed.leaseGeneration },- 'Obsolete local intent',- 'LOCAL_SUPERSEDED'- );- if (generation !== storageGeneration) return;- attemptInFlight = false;- resolveLiveOperationsAsQueued();- scheduleDrain();- return;- }- } catch {- queuePreparation = undefined;- try {- if (generation !== storageGeneration) return;- const live = liveQueuedOps.get(claimed.transactionId);- const operation =- live?.operation ??- makeOperation(- 'mutation',- createRequest(- replayDocument(claimed.query, claimed.operationName),- claimed.variables- ),- { url: '', requestPolicy: 'network-only' }- );- const failure = new CombinedError({- graphQLErrors: [- {- message: 'Unable to prepare the saved mutation for replay',- extensions: { code: 'LOCAL_RECOVERY_FAILED' },- },- ],- });- const result: OperationResult = {- operation,- data: undefined,- error: failure,- stale: false,- hasNext: false,- };- await recordAttemptResult(attempt, result, false);- await host.rollbackOptimisticWrite(- claimed.transactionId,- { owner: queueOwner, generation: claimed.leaseGeneration },- failure.message,- 'LOCAL_RECOVERY_FAILED'- );- liveQueuedOps.delete(claimed.transactionId);- live?.resolveRoute(- withOptimisticMutationDisposition(result, {- kind: 'permanently-failed',- transactionId: claimed.transactionId,- })- );- deferredUntil = undefined;- } finally {- if (generation === storageGeneration) {- attemptInFlight = false;- resolveLiveOperationsAsQueued();- scheduleDrain();- }- }- return;- }- if (generation !== storageGeneration) return;- deferredUntil = undefined;- // A superseded create can already exist on the server. Replay it to- // recover its identity before sending the newer edit or discard.- // Older hosts omit this flag: conservatively replay rather than loop- // forever asking the engine to discard a write it must retain.- if (claimed.superseded && claimed.requiresConfirmation === false) {- attemptInFlight = false;- const discarded = await host.deferOptimisticWrite(- claimed.transactionId,- { owner: queueOwner, generation: claimed.leaseGeneration },- Date.now(),- 'superseded before network replay'- );- if (discarded.kind !== 'discarded-superseded') {- throw new Error('superseded mutation was unexpectedly deferred');- }- const live = liveQueuedOps.get(claimed.transactionId);- if (live) {- liveQueuedOps.delete(claimed.transactionId);- live.resolveRoute(- queuedMutationResult(live.operation, claimed.transactionId)- );- }- resolveLiveOperationsAsQueued();- scheduleDrain();- return;- }- attemptInFlight = true;- const live = liveQueuedOps.get(claimed.transactionId);- if (live) {- liveQueuedOps.delete(claimed.transactionId);- live.resolveRoute(undefined);- enqueueForward(- withQueueRequestTimeout(- makeOperation(live.operation.kind, live.operation, {- ...live.operation.context,- [QUEUE_ATTEMPT_CONTEXT_KEY]: attempt,- })- )- );- } else {- let replayOperation: Operation | undefined;- try {- const replay = client.mutation(- replayDocument(claimed.query, claimed.operationName),- claimed.variables,- {- requestPolicy: 'network-only',- [QUEUE_ATTEMPT_CONTEXT_KEY]: attempt,- }- );- await replay.toPromise().then((result) => {- replayOperation = result.operation;- // Normal exchange results settle the attempt in writeThrough- // before resolving. Reject only an otherwise-unhandled error.- if (result.error && attemptInFlight) {- return Promise.reject(result.error);- }- });- } catch (error) {- try {- if (generation !== storageGeneration) return;- const rolledBack = await host.rollbackOptimisticWrite(- claimed.transactionId,- {- owner: queueOwner,- generation: claimed.leaseGeneration,- },- error instanceof Error ? error.message : String(error),- error instanceof CombinedError- ? mutationErrorCode(error)- : undefined- );- if (generation !== storageGeneration) return;- if (rolledBack.kind === 'rolled-back') {- void revalidateAfterSettlement(- rolledBack.revalidations ?? [],- replayOperation- );- }- } finally {- if (generation === storageGeneration) {- attemptInFlight = false;- scheduleDrain();- }- }- }- }- // Any other live operation is ordered behind the claimed head.- resolveLiveOperationsAsQueued();- }-- async function drainQueue(): Promise<void> {- if (attemptInFlight) {- // Every newly enqueued operation is behind the claimed head. Its- // caller can stop waiting; durable replay now owns the mutation.- resolveLiveOperationsAsQueued();- return;- }- if (drainRunning) return;- drainRunning = true;- drainRequested = false;- try {- // Restore can invalidate the generation while the host initializes.- // Finish that handshake before tagging a claim or starting its lease.- await host.currentRevision();- if (options.prepareMutationQueue) await prepareQueueRecovery();- const now = Date.now();- // A wakeup probes immediately, but must retain a future retry- // deadline if the durable head is not eligible yet. Consume expired- // hints so a head leased by another runner cannot cause a busy loop.- if (deferredUntil !== undefined && deferredUntil <= now) {- deferredUntil = undefined;- }- const generation = storageGeneration;- const claimed = await host.claimNextMutation(- queueOwner,- now,- now + QUEUE_LEASE_MS- );- if (!claimed) {- resolveLiveOperationsAsQueued();- // Short retries keep their deadline; uncertain five-minute lease- // backoffs must not prevent regular polling after a wakeup.- scheduleDrain(- deferredUntil === undefined- ? EMPTY_QUEUE_POLL_MS- : Math.min(- EMPTY_QUEUE_POLL_MS,- Math.max(0, deferredUntil - Date.now())- )- );- return;- }-- await routeClaimedMutation(claimed, generation);- } catch {- // Enqueue already succeeded, so callers must observe these as- // queued even if the runner cannot currently inspect the head.- resolveLiveOperationsAsQueued();- scheduleDrain(EMPTY_QUEUE_POLL_MS);- } finally {- drainRunning = false;- if (drainRequested) scheduleDrain();- }- }-- async function prepareQueueRecovery(): Promise<void> {- try {- queuePreparation ??= boundedMutationLifecycle(() =>- options.prepareMutationQueue!()- );- await queuePreparation;- } catch (error) {- queuePreparation = undefined;- console.warn(- '[graphql-cache] Unable to prepare mutation recovery',- error- );- }- }-- async function recordAttemptResult(- attempt: QueueAttemptContext,- result: OperationResult,- retry: boolean- ): Promise<void> {- if (!attempt.mutation || !options.onMutationAttemptResult) return;- try {- await boundedMutationLifecycle(() =>- options.onMutationAttemptResult!(attempt.mutation!, result, retry)- );- } catch (error) {- try {- options.onCacheError?.(error, result.operation);- } catch {- /* Diagnostics cannot retain a failed queue head. */- }- }- }async function revalidateAfterSettlement(revalidations: QueryRevalidationWire[],@@ -1443,7 +1011,7 @@ export function normalizedCacheExchange(async function prepareMutation(op: Operation): Promise<OperationResult | undefined> {- const generation = storageGeneration;+ const epoch = mutationQueue.epoch();if (host.disabled) {notifyOptimisticMutationEnqueued(op);enqueueForward(op);@@ -1481,7 +1049,7 @@ export function normalizedCacheExchange(try {enqueue = await host.enqueueOptimisticMutation(args, claim);} catch (error) {- if (generation !== storageGeneration)+ if (!mutationQueue.isCurrent(epoch))return uncertainEnqueueResult(op,new Error('cache storage changed during enqueue', {@@ -1541,7 +1109,7 @@ export function normalizedCacheExchange(claim);} catch (fallbackError) {- if (generation !== storageGeneration)+ if (!mutationQueue.isCurrent(epoch))return uncertainEnqueueResult(op,new Error('cache storage changed during enqueue', {@@ -1556,56 +1124,13 @@ export function normalizedCacheExchange(return undefined;}}- if (generation !== storageGeneration) {+ if (!mutationQueue.isCurrent(epoch)) {return uncertainEnqueueResult(op,new Error('cache storage changed during enqueue'));}- if (enqueue.upsertKind.kind === 'replaced-pending') {- const superseded = liveQueuedOps.get(- enqueue.upsertKind.removedTransactionId- );- if (superseded) {- liveQueuedOps.delete(enqueue.upsertKind.removedTransactionId);- superseded.resolveRoute(- queuedMutationResult(- superseded.operation,- enqueue.upsertKind.removedTransactionId- )- );- }- }- const routed = new Promise<OperationResult | undefined>((resolve) => {- liveQueuedOps.set(enqueue.transactionId, {- operation: op,- resolveRoute: resolve,- });- });- notifyOptimisticMutationEnqueued(op);- try {- await match(enqueue.initialClaim)- .with({ kind: 'claimed' }, ({ mutation }) =>- routeClaimedMutation(mutation, generation)- )- .with({ kind: 'not-runnable' }, () => {- resolveLiveOperationsAsQueued();- scheduleDrain();- })- .with({ kind: 'failed' }, ({ error }) => {- options.onCacheError?.(new Error(error), op);- resolveLiveOperationsAsQueued();- scheduleDrain(EMPTY_QUEUE_POLL_MS);- })- .exhaustive();- } catch (error) {- // Enqueue already succeeded. Preserve durable-runner ownership even- // if routing or claim rollback fails unexpectedly.- options.onCacheError?.(error, op);- resolveLiveOperationsAsQueued();- scheduleDrain(EMPTY_QUEUE_POLL_MS);- }- return await routed;+ return await mutationQueue.admit(op, enqueue, epoch);}/** Applies operation cache effects serially and isolates every failure. */@@ -1880,231 +1405,7 @@ export function normalizedCacheExchange(});} else if (op.kind === 'mutation') {const attempt = queueAttemptOf(op);- if (attempt) {- const isCurrent = () =>- attempt.storageGeneration === storageGeneration;- const detachedResult = () =>- withOptimisticMutationDisposition(result, {- kind:- result.data != null && !result.error- ? 'committed'- : 'permanently-failed',- transactionId: attempt.transactionId,- });- if (!isCurrent()) return detachedResult();- const claim = {- owner: attempt.leaseOwner,- generation: attempt.leaseGeneration,- };- let retryAt: number | undefined;- let settled = false;- let replacementTransactionId: string | undefined;- let disposition:- | 'committed'- | 'queued'- | 'superseded'- | 'permanently-failed' = 'queued';- try {- const response = result.error?.response ?? attempt.response;- const httpServerFailure = (response?.status ?? 0) >= 500;- // A valid GraphQL payload can arrive over HTTP 5xx without an- // urql error. Never commit it or bypass the retry policy.- if (httpServerFailure && !result.error) {- result = {- ...result,- data: undefined,- error: new CombinedError({- networkError: new Error(`HTTP ${response.status}`),- response,- }),- };- }- if (result.error || result.data == null) {- let retry = false;- if (result.error && options.shouldRetryMutation) {- try {- retry = await options.shouldRetryMutation(result.error);- } catch (error) {- options.onCacheError?.(error, op);- }- }- if (!isCurrent()) return detachedResult();- // urql represents HTTP 5xx as networkError too. Count actual- // server responses, never connection failures or timeouts.- const serverFailure =- httpServerFailure ||- (result.error?.graphQLErrors.length ?? 0) > 0;- if (- retry &&- serverFailure &&- attempt.serverFailureCount + 1 >= MAX_MUTATION_SERVER_FAILURES- ) {- retry = false;- result = {- ...result,- error: new CombinedError({- graphQLErrors: [- {- message: `Mutation stopped after ${MAX_MUTATION_SERVER_FAILURES} server failures: ${result.error?.message}`,- extensions: { code: 'MUTATION_RETRY_EXHAUSTED' },- },- ],- response: result.error?.response,- }),- };- }- await recordAttemptResult(attempt, result, retry);- if (!isCurrent()) return detachedResult();- if (retry) {- retryAt = Date.now() + retryDelayMs(attempt.attemptCount);- const deferred = await host.deferOptimisticWrite(- attempt.transactionId,- claim,- retryAt,- result.error?.message ?? 'mutation returned no data',- serverFailure- );- if (!isCurrent()) return detachedResult();- settled = true;- if (deferred.kind === 'discarded-superseded') {- retryAt = undefined;- replacementTransactionId =- deferred.replacementTransactionId;- disposition = 'superseded';- } else {- disposition = 'queued';- }- } else {- const rolledBack = await host.rollbackOptimisticWrite(- attempt.transactionId,- claim,- result.error?.message ?? 'mutation returned no data',- mutationErrorCode(result.error)- );- if (!isCurrent()) return detachedResult();- settled = true;- if (rolledBack.kind === 'discarded-superseded') {- replacementTransactionId =- rolledBack.replacementTransactionId;- disposition = 'superseded';- } else {- disposition = 'permanently-failed';- void revalidateAfterSettlement(- rolledBack.revalidations ?? [],- op- );- }- }- } else {- confirmedMutation = {- transactionId: attempt.transactionId,- result,- };- await recordAttemptResult(attempt, result, false);- if (!isCurrent()) return detachedResult();- const committed = await host.commitOptimisticWrite(- attempt.transactionId,- claim,- {- query: queryText(op),- operationName: operationName(op),- variables: op.variables as- | Record<string, unknown>- | undefined,- data: result.data,- }- );- if (!isCurrent()) return detachedResult();- settled = true;- confirmedMutation = undefined;- for (const error of committed.identityErrors ?? []) {- try {- options.onCacheError?.(new Error(error), op);- } catch {- // A diagnostic callback cannot undo confirmed settlement.- }- }- if (committed.kind === 'failed') {- const error = new Error(committed.error);- try {- options.onCacheError?.(error, op);- } catch {- // A diagnostic callback cannot undo confirmed settlement.- }- result = {- ...result,- data: undefined,- error: new CombinedError({ networkError: error }),- };- replacementTransactionId = committed.replacementTransactionId;- disposition = replacementTransactionId- ? 'superseded'- : 'permanently-failed';- if (!replacementTransactionId) {- void revalidateAfterSettlement(- committed.revalidations ?? [],- op- );- }- } else if (committed.kind === 'committed-superseded') {- replacementTransactionId = committed.replacementTransactionId;- disposition = 'superseded';- await deleteReportedRecords(result);- } else {- disposition = 'committed';- await deleteReportedRecords(result);- if (!isCurrent()) return detachedResult();- const effects = operationCacheEffects(result.data);- if (effects.some((effect) => effect.kind === 'delete')) {- // Commit already normalized the complete result. Replay only- // mixed explicit effects so their final write/delete order is- // identical to a non-optimistic operation.- await applyOperationCacheEffects(op, effects, isCurrent);- }- if (isCurrent())- await revalidateAfterSettlement(- committed.revalidations ?? [],- op- );- }- }- } catch (error) {- try {- options.onCacheError?.(error, op);- } catch {- // Reporting cannot discard the retained response or stop retries.- }- if (!settled && isCurrent()) {- retryAt = Date.now() + QUEUE_LEASE_MS;- // A lost acknowledgement may hide a completed transaction.- // Preserve optimism and let a fresh durable claim decide.- disposition = 'queued';- }- } finally {- if (isCurrent()) {- liveQueuedOps.delete(attempt.transactionId);- attemptInFlight = false;- deferredUntil = retryAt;- scheduleDrain(- retryAt === undefined ? 0 : Math.max(0, retryAt - Date.now())- );- }- }- return withOptimisticMutationDisposition(- result,- disposition === 'superseded'- ? {- kind: 'superseded',- transactionId: attempt.transactionId,- replacementTransactionId:- replacementTransactionId ?? attempt.transactionId,- }- : {- kind: disposition,- transactionId: attempt.transactionId,- }- );- }+ if (attempt) return await mutationQueue.settle(result, attempt);const optimistic = optimisticContextOf(op);if (result.data != null && !result.error) {@@ -2184,25 +1485,19 @@ export function normalizedCacheExchange();if (!host.disabled) {- scheduleDrain();+ mutationQueue.start();// Includes BFCache restoration, even with no active query keys. The// host gates claims on initialization; durable leases still decide// which head is runnable after reconnecting.host.onCacheGenerationChanged((change) => {cacheGeneration += 1;- if (change.storage === 'reset') {- storageGeneration += 1;- confirmedMutation = undefined;- attemptInFlight = false;- deferredUntil = undefined;- resolveLiveOperationsAsQueued();- }+ if (change.storage === 'reset') mutationQueue.resetStorage();liveQueries.clear();documentReader.clear();- wakeDrain();+ mutationQueue.wake();});if (typeof addEventListener === 'function') {- addEventListener('online', wakeDrain);+ addEventListener('online', mutationQueue.wake);}}void unsubscribePush;