Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
Seanaye/feat/purge queue (#7635)
Sean Ayea6d16828fc05
diff --git a/apps/web/docs/graphql-normalized-cache-plan.md b/apps/web/docs/graphql-normalized-cache-plan.mdindex 9251cf12eb..0c936f8d89 100644--- a/apps/web/docs/graphql-normalized-cache-plan.md+++ b/apps/web/docs/graphql-normalized-cache-plan.md@@ -100,8 +100,15 @@ entire cache in browser memory. With 10s of thousands of cached objects13. **Durable optimistic mutations** — optimistic GraphQL mutations arepersisted with their replay request, restored across restarts, and appliedstrictly in enqueue order. Retryability is decided by an exchange callback;- retryable failures retain their optimistic layer. Each queued network- attempt has a one-minute timeout, comfortably inside its five-minute lease.+ retryable failures retain their optimistic layer. Ten retryable server+ failures (GraphQL errors or HTTP 5xx responses) permanently fail the mutation+ through ordinary rollback so the next queued mutation can run. Transport+ failures/timeouts do not consume or reset that budget. The separate server+ failure counter survives reloads and tab handoff in pending-mutation metadata;+ `attempt_count` still counts every claim for backoff and coalescing safety.+ Settlement removes the pending mutation and counter; there is no retained DLQ.+ Each queued network attempt has a one-minute timeout, comfortably inside its+ five-minute lease.### Open questionsdiff --git a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.tsindex 016120b5ae..b2a02bc03b 100644--- a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts+++ b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts@@ -12,7 +12,9 @@ import { shouldRetryGraphqlMutation } from '@service-storage/graphql-mutation-reimport {type Client,CombinedError,+ createClient,createRequest,+ fetchExchange,gql,makeOperation,type Operation,@@ -49,6 +51,7 @@ import {normalizedCacheResultMetadata,} from './normalized-cache-exchange';import {+ executeOptimisticMutation,optimisticContextOf,optimisticMutationDispositionOf,} from './optimistic';@@ -205,7 +208,8 @@ type FakeHost = CacheHost & {teardowns: number[];scriptRead: (result: ReadResult) => void;seedQueued: (- args: Parameters<CacheHost['enqueueOptimisticMutation']>[0]+ args: Parameters<CacheHost['enqueueOptimisticMutation']>[0],+ serverFailureCount?: number) => void;pushAffected: (opKeys: number[]) => void;pushGeneration: (change: CacheGenerationChange) => void;@@ -221,6 +225,7 @@ function makeFakeHost(): FakeHost {transactionId: string;args: Parameters<CacheHost['enqueueOptimisticMutation']>[0];attemptCount: number;+ serverFailureCount: number;leased: boolean;nextAttemptAtMs?: number;}> = [];@@ -248,6 +253,7 @@ function makeFakeHost(): FakeHost {operationName: head.args.operationName,variables: head.args.variables ?? {},attemptCount: head.attemptCount,+ serverFailureCount: head.serverFailureCount,};}@@ -266,11 +272,12 @@ function makeFakeHost(): FakeHost {scriptRead: (r) => {readResult = r;},- seedQueued: (args) => {+ seedQueued: (args, serverFailureCount = 0) => {queue.push({transactionId: `restored-${queue.length + 1}`,args,attemptCount: 0,+ serverFailureCount,leased: false,});},@@ -341,7 +348,13 @@ function makeFakeHost(): FakeHost {linkPatches: args.linkPatches,});const transactionId = `txn-${host.begins.length}`;- queue.push({ transactionId, args, attemptCount: 0, leased: false });+ queue.push({+ transactionId,+ args,+ attemptCount: 0,+ serverFailureCount: 0,+ leased: false,+ });const mutation = claimQueueHead(claim.nowMs);return {transactionId,@@ -372,13 +385,15 @@ function makeFakeHost(): FakeHost {transactionId,_claim: MutationClaim,nextAttemptAtMs,- error+ error,+ serverFailure = false) {host.defers.push({ transactionId, error });const head = queue[0];if (head?.transactionId === transactionId) {head.leased = false;head.nextAttemptAtMs = nextAttemptAtMs;+ if (serverFailure) head.serverFailureCount += 1;}return { kind: 'deferred' };},@@ -4002,6 +4017,285 @@ describe('normalizedCacheExchange', () => {});});+ it.each([+ { status: 500, data: null, failures: 0 },+ { status: 503, data: { from: 'server' }, failures: 0 },+ { status: 500, data: null, failures: 9 },+ { status: 503, data: { from: 'server' }, failures: 9 },+ ])(+ 'handles HTTP $status without GraphQL errors after $failures server failures',+ async ({ status, data, failures }) => {+ vi.useFakeTimers();+ try {+ if (failures > 0) {+ host.seedQueued(+ {+ uuid: crypto.randomUUID(),+ query: stringifyDocument(MUTATION),+ data: optimistic,+ },+ failures+ );+ }+ const response = new Response(JSON.stringify({ data }), {+ status,+ headers: { 'content-type': 'application/json' },+ });+ const shouldRetryMutation = vi.fn(shouldRetryGraphqlMutation);+ const rollback = vi.spyOn(host, 'rollbackOptimisticWrite');+ const defer = vi.spyOn(host, 'deferOptimisticWrite');+ const client = createClient({+ url: 'http://test/graphql',+ fetch: vi.fn(async () => response),+ exchanges: [+ normalizedCacheExchange(host, { shouldRetryMutation }),+ fetchExchange,+ ],+ });+ const pending =+ failures === 0+ ? executeOptimisticMutation(+ client,+ MUTATION,+ { input: {} },+ optimistic,+ { uuid: crypto.randomUUID() }+ ).toPromise()+ : undefined;+ await vi.advanceTimersByTimeAsync(10);+ expect(shouldRetryMutation).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({ response, graphQLErrors: [] })+ );+ expect(host.commits).toHaveLength(0);+ if (failures === 0) {+ expect(defer).toHaveBeenCalledExactlyOnceWith(+ 'txn-1',+ expect.any(Object),+ expect.any(Number),+ expect.stringContaining(String(status)),+ true+ );+ expect(host.rollbacks).toHaveLength(0);+ expect(optimisticMutationDispositionOf((await pending)!)).toEqual({+ kind: 'queued',+ transactionId: 'txn-1',+ });+ } else {+ expect(host.defers).toHaveLength(0);+ expect(rollback).toHaveBeenCalledExactlyOnceWith(+ 'restored-1',+ expect.any(Object),+ expect.stringContaining('10 server failures'),+ 'MUTATION_RETRY_EXHAUSTED'+ );+ }+ } finally {+ vi.clearAllTimers();+ vi.useRealTimers();+ }+ }+ );++ it.each([false, true])(+ 'preserves success and explicit GraphQL rejection with fetchExchange (rejected=%s)',+ async (rejected) => {+ vi.useFakeTimers();+ try {+ const response = new Response(+ JSON.stringify(+ rejected+ ? {+ errors: [+ { message: 'rejected', extensions: { code: 'INVALID' } },+ ],+ }+ : { data: optimistic }+ ),+ {+ status: rejected ? 503 : 200,+ headers: { 'content-type': 'application/json' },+ }+ );+ const client = createClient({+ url: 'http://test/graphql',+ fetch: vi.fn(async () => response),+ exchanges: [+ normalizedCacheExchange(host, {+ shouldRetryMutation: shouldRetryGraphqlMutation,+ }),+ fetchExchange,+ ],+ });+ const pending = executeOptimisticMutation(+ client,+ MUTATION,+ { input: {} },+ optimistic,+ { uuid: crypto.randomUUID() }+ ).toPromise();+ await vi.advanceTimersByTimeAsync(10);+ const result = await pending;+ expect(host.defers).toHaveLength(0);+ expect(optimisticMutationDispositionOf(result)?.kind).toBe(+ rejected ? 'permanently-failed' : 'committed'+ );+ if (rejected) {+ expect(result.error?.graphQLErrors[0]?.extensions.code).toBe(+ 'INVALID'+ );+ expect(result.error?.response).toBe(response);+ }+ } finally {+ vi.clearAllTimers();+ vi.useRealTimers();+ }+ }+ );++ it.each(['graphql', 'http-500', 'http-503'])(+ 'permanently fails the tenth %s failure and advances the queue',+ async (kind) => {+ vi.useFakeTimers();+ try {+ const error =+ kind === 'graphql'+ ? new CombinedError({+ graphQLErrors: [+ {+ message: 'persistent failure',+ extensions: { retryable: true },+ },+ ],+ })+ : new CombinedError({+ networkError: new Error('server unavailable'),+ response: new Response(null, {+ status: kind === 'http-500' ? 500 : 503,+ }),+ });+ const rollback = vi.spyOn(host, 'rollbackOptimisticWrite');+ let attempts = 0;+ const { ops, results, forwarded } = harness(+ host,+ () =>+ ++attempts <= 10+ ? { error, data: undefined }+ : { data: optimistic },+ { shouldRetryMutation: shouldRetryGraphqlMutation }+ );+ ops.next(makeMutationOp(1, optimistic));+ ops.next(makeMutationOp(2, optimistic));+ await vi.advanceTimersByTimeAsync(0);+ for (let attempt = 1; attempt < 10; attempt += 1) {+ expect(forwarded).toHaveLength(attempt);+ expect(host.rollbacks).toHaveLength(0);+ await vi.advanceTimersByTimeAsync(+ Math.min(1_000 * 2 ** (attempt - 1), 60_000)+ );+ }+ await vi.advanceTimersByTimeAsync(1);+ expect(host.defers).toHaveLength(9);+ expect(rollback).toHaveBeenCalledExactlyOnceWith(+ 'txn-1',+ expect.any(Object),+ expect.stringContaining('10 server failures'),+ 'MUTATION_RETRY_EXHAUSTED'+ );+ expect(host.commits.map((entry) => entry.transactionId)).toEqual([+ 'txn-2',+ ]);+ expect(forwarded).toHaveLength(11);+ expect(+ results.some(+ (result) =>+ optimisticMutationDispositionOf(result)?.kind ===+ 'permanently-failed'+ )+ ).toBe(true);+ } finally {+ vi.clearAllTimers();+ vi.useRealTimers();+ }+ }+ );++ it('network failures neither consume nor reset the server failure budget', async () => {+ vi.useFakeTimers();+ try {+ const server = new CombinedError({+ graphQLErrors: [+ { message: 'server failed', extensions: { retryable: true } },+ ],+ });+ const network = new CombinedError({+ networkError: new DOMException('request timed out', 'TimeoutError'),+ });+ let attempts = 0;+ const { ops } = harness(+ host,+ () => {+ attempts += 1;+ // Transport failures before the first server error cannot consume+ // the budget; transport failures after nine cannot reset it either.+ return {+ error:+ (attempts >= 13 && attempts <= 21) || attempts === 34+ ? server+ : network,+ data: undefined,+ };+ },+ { shouldRetryMutation: shouldRetryGraphqlMutation }+ );+ ops.next(makeMutationOp(1, optimistic));+ await vi.advanceTimersByTimeAsync(0);+ for (let attempt = 1; attempt < 34; attempt += 1) {+ expect(host.rollbacks).toHaveLength(0);+ await vi.advanceTimersByTimeAsync(+ Math.min(1_000 * 2 ** (attempt - 1), 60_000)+ );+ }+ expect(attempts).toBe(34);+ expect(host.defers).toHaveLength(33);+ expect(host.rollbacks).toEqual(['txn-1']);+ } finally {+ vi.clearAllTimers();+ vi.useRealTimers();+ }+ });++ it('uses the persisted budget on replay and revalidates after exhaustion', async () => {+ host.seedQueued(+ {+ uuid: crypto.randomUUID(),+ query: stringifyDocument(MUTATION),+ data: optimistic,+ revalidations: [+ { query: stringifyDocument(QUERY), variablesJson: '{}' },+ ],+ },+ 9+ );+ const rollback = vi.spyOn(host, 'rollbackOptimisticWrite');+ const error = new CombinedError({+ graphQLErrors: [+ { message: 'still failing', extensions: { retryable: true } },+ ],+ });+ const { client } = harness(host, () => ({ error, data: undefined }), {+ shouldRetryMutation: shouldRetryGraphqlMutation,+ });+ await tick();+ expect(host.defers).toHaveLength(0);+ expect(rollback).toHaveBeenCalledExactlyOnceWith(+ 'restored-1',+ expect.any(Object),+ expect.stringContaining('10 server failures'),+ 'MUTATION_RETRY_EXHAUSTED'+ );+ expect(client.query).toHaveBeenCalledOnce();+ });+it('retries a draft response failure with the same handles and commits after recovery', async () => {vi.useFakeTimers();try {diff --git a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.tsindex 9205342eaa..cd20d27015 100644--- a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts+++ b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts@@ -23,8 +23,9 @@* Mutations:* - With an optimistic response (see `executeOptimisticMutation`): the* mutation and layer are durably queued before the ordered runner forwards- * it. Retryable failures retain optimism for background replay; permanent- * failures roll back. A caller whose operation is blocked behind the queue+ * it. Retryable failures retain optimism for background replay, but the tenth+ * server failure permanently rolls back; transport failures are not counted.+ * Other permanent failures roll back immediately. A caller blocked behind the queue* head receives a synthetic `queued` disposition instead of waiting.* - Without one: forwarded normally; successful responses are normalized* through the standard write path so dependent cached queries update.@@ -107,6 +108,7 @@ const HYDRATION_DOCUMENT_CONTEXT_KEY = 'normalizedCacheHydrationDocument';const QUEUE_REQUEST_TIMEOUT_MS = 60_000;const QUEUE_LEASE_MS = 5 * 60_000;const EMPTY_QUEUE_POLL_MS = 30_000;+const MAX_MUTATION_SERVER_FAILURES = 10;const NORMALIZED_CACHE_RESULT_METADATA_KEY = '__macroNormalizedCache';@@ -169,6 +171,9 @@ type QueueAttemptContext = {leaseOwner: string;leaseGeneration: string;attemptCount: number;+ serverFailureCount: number;+ /** Transport-only metadata; urql drops the response on error-free payloads. */+ response?: Response;};function queueAttemptOf(op: Operation): QueueAttemptContext | undefined {@@ -273,19 +278,22 @@ function retryDelayMs(attemptCount: number): number {return Math.min(1_000 * 2 ** Math.max(0, attemptCount - 1), 60_000);}-/** Applies a hard network bound well inside the durable queue lease. */+/** Bounds network time inside the lease and retains HTTP status for settlement. */function withQueueRequestTimeout(op: Operation): Operation {const operationFetch = op.context.fetch ?? globalThis.fetch;+ const attempt = queueAttemptOf(op);return makeOperation(op.kind, op, {...op.context,- fetch: (input, init) => {+ fetch: async (input, init) => {const timeoutSignal = AbortSignal.timeout(QUEUE_REQUEST_TIMEOUT_MS);- return operationFetch(input, {+ const response = await operationFetch(input, {...init,signal: init?.signal? AbortSignal.any([init.signal, timeoutSignal]): timeoutSignal,});+ if (attempt) attempt.response = response;+ return response;},});}@@ -863,6 +871,7 @@ export function normalizedCacheExchange(leaseOwner: queueOwner,leaseGeneration: claimed.leaseGeneration,attemptCount: claimed.attemptCount,+ serverFailureCount: claimed.serverFailureCount ?? 0,};const live = liveQueuedOps.get(claimed.transactionId);if (live) {@@ -1473,6 +1482,20 @@ export function normalizedCacheExchange(| 'superseded'| 'permanently-failed' = 'queued';try {+ const response = result.error?.response ?? attempt.response;+ const httpServerFailure = (response?.status ?? 0) >= 500;+ // A valid GraphQL payload can arrive over HTTP 5xx without an+ // urql error. Never commit it or bypass the retry policy.+ if (httpServerFailure && !result.error) {+ result = {+ ...result,+ data: undefined,+ error: new CombinedError({+ networkError: new Error(`HTTP ${response.status}`),+ response,+ }),+ };+ }if (result.error || result.data == null) {let retry = false;if (result.error && options.shouldRetryMutation) {@@ -1482,13 +1505,38 @@ export function normalizedCacheExchange(options.onCacheError?.(error, op);}}+ // urql represents HTTP 5xx as networkError too. Count actual+ // server responses, never connection failures or timeouts.+ const serverFailure =+ httpServerFailure ||+ (result.error?.graphQLErrors.length ?? 0) > 0;+ if (+ retry &&+ serverFailure &&+ attempt.serverFailureCount + 1 >= MAX_MUTATION_SERVER_FAILURES+ ) {+ retry = false;+ result = {+ ...result,+ error: new CombinedError({+ graphQLErrors: [+ {+ message: `Mutation stopped after ${MAX_MUTATION_SERVER_FAILURES} server failures: ${result.error?.message}`,+ extensions: { code: 'MUTATION_RETRY_EXHAUSTED' },+ },+ ],+ response: result.error?.response,+ }),+ };+ }if (retry) {retryAt = Date.now() + retryDelayMs(attempt.attemptCount);const deferred = await host.deferOptimisticWrite(attempt.transactionId,claim,retryAt,- result.error?.message ?? 'mutation returned no data'+ result.error?.message ?? 'mutation returned no data',+ serverFailure);if (deferred.kind === 'discarded-superseded') {retryAt = undefined;diff --git a/apps/web/src/lib/graphql-cache/host/retirable-host.test.ts b/apps/web/src/lib/graphql-cache/host/retirable-host.test.tsindex fc0c01d13b..48a132d59b 100644--- a/apps/web/src/lib/graphql-cache/host/retirable-host.test.ts+++ b/apps/web/src/lib/graphql-cache/host/retirable-host.test.ts@@ -75,6 +75,28 @@ describe('createRetirableCacheHost', () => {expect(inner.teardown).not.toHaveBeenCalled();});+ it('forwards retry accounting and permanent-failure codes', async () => {+ const inner = createNoopCacheHost();+ const defer = vi.spyOn(inner, 'deferOptimisticWrite');+ const rollback = vi.spyOn(inner, 'rollbackOptimisticWrite');+ const host = createRetirableCacheHost(inner);+ const claim = { owner: 'runner', generation: '2' };+ await host.deferOptimisticWrite('1', claim, 100, 'server failed', true);+ expect(defer).toHaveBeenCalledWith('1', claim, 100, 'server failed', true);+ await host.rollbackOptimisticWrite(+ '1',+ claim,+ 'exhausted',+ 'MUTATION_RETRY_EXHAUSTED'+ );+ expect(rollback).toHaveBeenCalledWith(+ '1',+ claim,+ 'exhausted',+ 'MUTATION_RETRY_EXHAUSTED'+ );+ });+it('retires once', () => {const inner = disposedLikeHost();const host = createRetirableCacheHost(inner);diff --git a/apps/web/src/lib/graphql-cache/host/retirable-host.ts b/apps/web/src/lib/graphql-cache/host/retirable-host.tsindex f0c1946248..7b7ea957fb 100644--- a/apps/web/src/lib/graphql-cache/host/retirable-host.ts+++ b/apps/web/src/lib/graphql-cache/host/retirable-host.ts@@ -30,17 +30,24 @@ export function createRetirableCacheHost(inner: CacheHost): CacheHost {inspectQuery: (args) => current.inspectQuery(args),claimNextMutation: (owner, nowMs, leaseExpiresAtMs) =>current.claimNextMutation(owner, nowMs, leaseExpiresAtMs),- deferOptimisticWrite: (transactionId, claim, nextAttemptAtMs, error) =>+ deferOptimisticWrite: (+ transactionId,+ claim,+ nextAttemptAtMs,+ error,+ serverFailure+ ) =>current.deferOptimisticWrite(transactionId,claim,nextAttemptAtMs,- error+ error,+ serverFailure),commitOptimisticWrite: (transactionId, claim, args) =>current.commitOptimisticWrite(transactionId, claim, args),- rollbackOptimisticWrite: (transactionId, claim, error) =>- current.rollbackOptimisticWrite(transactionId, claim, error),+ rollbackOptimisticWrite: (transactionId, claim, error, errorCode) =>+ current.rollbackOptimisticWrite(transactionId, claim, error, errorCode),invalidate: (keys) => current.invalidate(keys),deleteRecords: (keys) => current.deleteRecords(keys),teardown: (opKey) => current.teardown(opKey),diff --git a/apps/web/src/lib/graphql-cache/host/tauri-host.ts b/apps/web/src/lib/graphql-cache/host/tauri-host.tsindex 3a398a911c..9941b529cf 100644--- a/apps/web/src/lib/graphql-cache/host/tauri-host.ts+++ b/apps/web/src/lib/graphql-cache/host/tauri-host.ts@@ -433,7 +433,8 @@ export function createTauriCacheHost(options: TauriHostOptions): CacheHost {transactionId: string,claim: MutationClaim,nextAttemptAtMs: number,- error: string+ error: string,+ serverFailure = false): Promise<DeferOptimisticWriteResult> {await ready;return await request<DeferOptimisticWriteResult>(@@ -444,6 +445,7 @@ export function createTauriCacheHost(options: TauriHostOptions): CacheHost {leaseGeneration: claim.generation,nextAttemptAtMs,error,+ serverFailure,});},diff --git a/apps/web/src/lib/graphql-cache/host/types.ts b/apps/web/src/lib/graphql-cache/host/types.tsindex 7cea57549c..08545b0799 100644--- a/apps/web/src/lib/graphql-cache/host/types.ts+++ b/apps/web/src/lib/graphql-cache/host/types.ts@@ -153,7 +153,8 @@ export interface CacheHost {transactionId: string,claim: MutationClaim,nextAttemptAtMs: number,- error: string+ error: string,+ serverFailure?: boolean): Promise<DeferOptimisticWriteResult>;/** Atomically commits a claimed mutation's real network response. */commitOptimisticWrite(diff --git a/apps/web/src/lib/graphql-cache/host/worker-host.ts b/apps/web/src/lib/graphql-cache/host/worker-host.tsindex b02039b267..0d358ae2fa 100644--- a/apps/web/src/lib/graphql-cache/host/worker-host.ts+++ b/apps/web/src/lib/graphql-cache/host/worker-host.ts@@ -1314,7 +1314,8 @@ export function createWorkerCacheHost(options: WorkerHostOptions): CacheHost {transactionId: string,claim: MutationClaim,nextAttemptAtMs: number,- error: string+ error: string,+ serverFailure = false) {return (await initializedRequest({kind: 'defer-optimistic-write',@@ -1323,6 +1324,7 @@ export function createWorkerCacheHost(options: WorkerHostOptions): CacheHost {leaseGeneration: claim.generation,nextAttemptAtMs,error,+ serverFailure,})) as DeferOptimisticWriteResult;},diff --git a/apps/web/src/lib/graphql-cache/protocol.ts b/apps/web/src/lib/graphql-cache/protocol.tsindex 9bf468a99f..deb4eb9de2 100644--- a/apps/web/src/lib/graphql-cache/protocol.ts+++ b/apps/web/src/lib/graphql-cache/protocol.ts@@ -417,6 +417,8 @@ export type ClaimedMutation = {/** Identity witness captured at enqueue time. */identity?: string;attemptCount: number;+ /** Retryable server failures; transport failures do not consume this budget. */+ serverFailureCount: number;};/** Outcome of the strict-head claim attempted immediately after enqueue. */@@ -554,6 +556,7 @@ export type CacheRequest = { id: number } & (leaseGeneration: string;nextAttemptAtMs: number;error: string;+ serverFailure?: boolean;}/** Atomically replace a claimed layer with the real network response. */| {diff --git a/apps/web/src/lib/graphql-cache/worker/browser-test/cache-lifecycle-harness.ts b/apps/web/src/lib/graphql-cache/worker/browser-test/cache-lifecycle-harness.tsindex de6dee4dba..c9a9724697 100644--- a/apps/web/src/lib/graphql-cache/worker/browser-test/cache-lifecycle-harness.ts+++ b/apps/web/src/lib/graphql-cache/worker/browser-test/cache-lifecycle-harness.ts@@ -1,9 +1,16 @@-import { type Client, createClient, gql, stringifyDocument } from '@urql/core';+import {+ type Client,+ createClient,+ fetchExchange,+ gql,+ stringifyDocument,+} from '@urql/core';import { filter, map, pipe } from 'wonka';import { normalizedCacheExchange } from '../../exchange/normalized-cache-exchange';import type { CacheHost } from '../../host/types';import { createWorkerCacheHost } from '../../host/worker-host';import { clearRegisteredCaches, registerCacheHost } from '../../lifecycle';+import type { MutationSettlement } from '../../protocol';const result = document.querySelector<HTMLElement>('#result');if (!result) throw new Error('missing Cache result node');@@ -227,6 +234,109 @@ const api = {);return queued.transactionId;},+ async seedRetryBudget(): Promise<string[]> {+ const host = requireOwner();+ const nowMs = Date.now();+ const owner = 'retry-budget-seed';+ const mutation = `mutation RetryBudget($input: SetEntityPropertyInput!) {+ setEntityProperty(input: $input) { id displayName }+ }`;+ const enqueue = (id: string) =>+ host.enqueueOptimisticMutation(+ {+ uuid: crypto.randomUUID(),+ query: mutation,+ variables: {+ input: {+ entityType: 'DOCUMENT',+ entityId: id,+ propertyDefinitionId: 'definition',+ value: { string: id },+ },+ },+ data: { setEntityProperty: { id, displayName: id } },+ },+ { owner, nowMs, leaseExpiresAtMs: nowMs + 300_000 }+ );+ const first = await enqueue('failing-property');+ const second = await enqueue('following-property');+ if (first.initialClaim.kind !== 'claimed')+ throw new Error('missing first claim');+ let claim = first.initialClaim.mutation;+ for (let failure = 0; failure < 9; failure += 1) {+ await host.deferOptimisticWrite(+ first.transactionId,+ { owner, generation: claim.leaseGeneration },+ nowMs,+ 'server failure',+ true+ );+ if (failure < 8) {+ const next = await host.claimNextMutation(+ owner,+ nowMs,+ nowMs + 300_000+ );+ if (!next) throw new Error('missing retry claim');+ claim = next;+ }+ }+ // Transport failure after nine server failures must preserve the budget.+ const network = await host.claimNextMutation(owner, nowMs, nowMs + 300_000);+ if (!network || network.serverFailureCount !== 9)+ throw new Error('budget not persisted');+ await host.deferOptimisticWrite(+ first.transactionId,+ { owner, generation: network.leaseGeneration },+ nowMs,+ 'offline',+ false+ );+ return [first.transactionId, second.transactionId];+ },+ async exhaustRetryBudget(): Promise<{+ attempts: number;+ settlements: MutationSettlement[];+ }> {+ const host = requireOwner();+ const complete = Promise.withResolvers<void>();+ const settlements: MutationSettlement[] = [];+ let attempts = 0;+ const unsubscribe = host.onMutationSettled((settlement) => {+ settlements.push(settlement);+ if (settlements.length === 2) complete.resolve();+ });+ mutationRunner = createClient({+ url: 'http://cache-lifecycle.test/graphql',+ fetch: async () => {+ attempts += 1;+ // Even a valid data payload over HTTP 5xx must exhaust the budget.+ return new Response(+ JSON.stringify({+ data: {+ setEntityProperty: {+ id: attempts === 1 ? 'failing-property' : 'following-property',+ displayName: 'Synced',+ },+ },+ }),+ {+ status: attempts === 1 ? 503 : 200,+ headers: { 'content-type': 'application/json' },+ }+ );+ },+ exchanges: [+ normalizedCacheExchange(host, {+ shouldRetryMutation: (error) => (error.response?.status ?? 0) >= 500,+ }),+ fetchExchange,+ ],+ });+ await complete.promise;+ unsubscribe();+ return { attempts, settlements };+ },navigationState() {return {clientId: requireOwner().clientId,diff --git a/apps/web/src/lib/graphql-cache/worker/browser-test/mutation-retry.browser.e2e.ts b/apps/web/src/lib/graphql-cache/worker/browser-test/mutation-retry.browser.e2e.tsnew file mode 100644index 0000000000..d4c3f0b196--- /dev/null+++ b/apps/web/src/lib/graphql-cache/worker/browser-test/mutation-retry.browser.e2e.ts@@ -0,0 +1,33 @@+import { expect, test } from '@playwright/test';++test('Cache exhausts a persisted server retry budget after reload and advances the queue', async ({+ page,+}, testInfo) => {+ const path = testInfo.project.name.includes('production')+ ? '/app/cache-lifecycle.html'+ : '/cache-lifecycle.html';+ const scope = `cache-retry-budget-${crypto.randomUUID()}`;+ await page.goto(`${path}?treatment=true&scope=${scope}`);+ await expect(page.locator('#result')).toHaveAttribute('data-status', 'ready');+ await page.evaluate(() => window.cacheLifecycleHarness.startSingle());+ const ids = await page.evaluate(() =>+ window.cacheLifecycleHarness.seedRetryBudget()+ );+ await page.evaluate(() => window.cacheLifecycleHarness.dispose());+ await page.reload();+ await expect(page.locator('#result')).toHaveAttribute('data-status', 'ready');+ await page.evaluate(() => window.cacheLifecycleHarness.startSingle());+ const result = await page.evaluate(() =>+ window.cacheLifecycleHarness.exhaustRetryBudget()+ );+ expect(result.attempts).toBe(2);+ expect(result.settlements).toMatchObject([+ {+ transactionId: ids[0],+ status: 'permanently-failed',+ errorCode: 'MUTATION_RETRY_EXHAUSTED',+ },+ { transactionId: ids[1], status: 'committed' },+ ]);+ await page.evaluate(() => window.cacheLifecycleHarness.dispose());+});diff --git a/apps/web/src/lib/graphql-cache/worker/browser-test/playwright.production.config.ts b/apps/web/src/lib/graphql-cache/worker/browser-test/playwright.production.config.tsindex 66dc8f73af..6381d38d2d 100644--- a/apps/web/src/lib/graphql-cache/worker/browser-test/playwright.production.config.ts+++ b/apps/web/src/lib/graphql-cache/worker/browser-test/playwright.production.config.ts@@ -12,6 +12,7 @@ export default defineConfig({'cache-artifact-compatibility.browser.e2e.ts','cache-wasm-packaging.browser.e2e.ts','cache-lifecycle.browser.e2e.ts',+ 'mutation-retry.browser.e2e.ts',],timeout: 90_000,fullyParallel: false,diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.tsindex 6b827da0ec..f51e380480 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts@@ -57,6 +57,24 @@ it.each([undefined, 'DRAFT_ALREADY_SENT', 'INTERNAL', 42, null])(});+it.each([undefined, true, false, 'true', 1, null])(+ 'validates retry-budget classification %j',+ (serverFailure) => {+ expect(+ isCacheRequest({+ id: 1,+ kind: 'defer-optimistic-write',+ transactionId: '1',+ leaseOwner: 'runner',+ leaseGeneration: '1',+ nextAttemptAtMs: 100,+ error: 'failed',+ serverFailure,+ })+ ).toBe(serverFailure === undefined || typeof serverFailure === 'boolean');+ }+);+const enginePort = {postMessage() {},close() {},diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.tsindex efe4448929..6d98860bbe 100644--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts@@ -623,12 +623,15 @@ export function isCacheRequest(value: unknown): value is CacheRequest {'leaseGeneration','nextAttemptAtMs','error',+ 'serverFailure',]) &&isString(value.transactionId) &&isString(value.leaseOwner) &&isString(value.leaseGeneration) &&isSafeNonNegativeInteger(value.nextAttemptAtMs) &&- isString(value.error)+ isString(value.error) &&+ (value.serverFailure === undefined ||+ typeof value.serverFailure === 'boolean'));case 'commit-optimistic-write':return (diff --git a/apps/web/src/lib/graphql-cache/worker/wasm-module.ts b/apps/web/src/lib/graphql-cache/worker/wasm-module.tsindex d27b53684b..476470bf80 100644--- a/apps/web/src/lib/graphql-cache/worker/wasm-module.ts+++ b/apps/web/src/lib/graphql-cache/worker/wasm-module.ts@@ -152,7 +152,8 @@ export interface CacheEngine {leaseOwner: string,leaseGeneration: string,nextAttemptAtMs: number,- error: string+ error: string,+ serverFailure: boolean): Promise<DeferOptimisticWriteResult>;commitOptimisticWrite(transactionId: string,diff --git a/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts b/apps/web/src/lib/graphql-cache/worker/worker-core.test.tsindex 33c0620907..c4eb34279e 100644--- a/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts+++ b/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts@@ -375,6 +375,37 @@ describe('CacheWorkerCore', () => {});});+ it.each([undefined, false, true])(+ 'forwards server-failure accounting to wasm: %s',+ async (serverFailure) => {+ const deferOptimisticWrite = vi.fn(async () => ({ kind: 'deferred' }));+ loadCacheWasmMock.mockResolvedValue({+ openCache: vi.fn().mockResolvedValue({ deferOptimisticWrite }),+ });+ const port = { postMessage: vi.fn() };+ const core = new CacheWorkerCore();+ await core.handleRequest(port, { id: 1, kind: 'init', scope: 'scope-1' });+ await core.handleRequest(port, {+ id: 2,+ kind: 'defer-optimistic-write',+ transactionId: '1',+ leaseOwner: 'runner',+ leaseGeneration: '2',+ nextAttemptAtMs: 100,+ error: 'failed',+ serverFailure,+ });+ expect(deferOptimisticWrite).toHaveBeenCalledExactlyOnceWith(+ '1',+ 'runner',+ '2',+ 100,+ 'failed',+ serverFailure ?? false+ );+ }+ );+it('runs standalone claims ahead of queued observational reads', async () => {const order: string[] = [];let releaseBlocker!: () => void;diff --git a/apps/web/src/lib/graphql-cache/worker/worker-core.ts b/apps/web/src/lib/graphql-cache/worker/worker-core.tsindex 782aac03f0..e7e7aff074 100644--- a/apps/web/src/lib/graphql-cache/worker/worker-core.ts+++ b/apps/web/src/lib/graphql-cache/worker/worker-core.ts@@ -613,7 +613,8 @@ export class CacheWorkerCore {request.leaseOwner,request.leaseGeneration,request.nextAttemptAtMs,- request.error+ request.error,+ request.serverFailure ?? false);if (result.kind === 'discarded-superseded') {result.revision = parseCacheRevision(result.revision);diff --git a/apps/web/tauri/graphql_cache_plugin/src/commands.rs b/apps/web/tauri/graphql_cache_plugin/src/commands.rsindex 44b96477bc..4af03abbf1 100644--- a/apps/web/tauri/graphql_cache_plugin/src/commands.rs+++ b/apps/web/tauri/graphql_cache_plugin/src/commands.rs@@ -408,6 +408,7 @@ pub async fn graphql_cache_defer_optimistic_write<R: Runtime>(lease_generation: String,next_attempt_at_ms: i64,error: String,+ server_failure: Option<bool>,) -> Result<DeferOptimisticWriteResultWire, String> {let settlement_transaction_id = transaction_id.clone();let result = engine_handle(&state)?@@ -417,6 +418,7 @@ pub async fn graphql_cache_defer_optimistic_write<R: Runtime>(lease_generation,next_attempt_at_ms,error,+ server_failure.unwrap_or(false),).await?;if let DeferOptimisticWriteResultWire::DiscardedSuperseded {diff --git a/apps/web/tauri/graphql_cache_plugin/src/engine.rs b/apps/web/tauri/graphql_cache_plugin/src/engine.rsindex 6599c0cc01..2b80e076cb 100644--- a/apps/web/tauri/graphql_cache_plugin/src/engine.rs+++ b/apps/web/tauri/graphql_cache_plugin/src/engine.rs@@ -205,6 +205,8 @@ pub struct ClaimedMutationWire {pub identity: Option<String>,/// Number of network attempts including this claim.pub attempt_count: u32,+ /// Retryable server failures, excluding transport failures.+ pub server_failure_count: u32,}/// Tagged result of deferring or discarding a failed queue attempt.@@ -303,6 +305,7 @@ impl TryFrom<ClaimedMutation> for ClaimedMutationWire {variables: serde_json::from_str(&request.variables_json).map_err(|e| e.to_string())?,identity: request.identity,attempt_count: claimed.queued.mutation.attempt_count,+ server_failure_count: claimed.queued.mutation.server_failure_count,})}}@@ -770,6 +773,7 @@ impl EngineHandle {lease_generation: String,next_attempt_at_ms: i64,error: String,+ server_failure: bool,) -> Result<DeferOptimisticWriteResultWire, String> {let transaction = parse_transaction_id(&transaction_id)?;let claim = MutationClaimToken {@@ -779,7 +783,13 @@ impl EngineHandle {let mut state = self.inner.lock().await;let EngineState { engine, ops, .. } = &mut *state;match engine- .defer_optimistic_write(transaction, claim, next_attempt_at_ms, error)+ .defer_optimistic_write(+ transaction,+ claim,+ next_attempt_at_ms,+ error,+ server_failure,+ ).await.map_err(|e| e.to_string())?{diff --git a/crates/client/README.md b/crates/client/README.mdindex 676955ab3f..78b3ab479b 100644--- a/crates/client/README.md+++ b/crates/client/README.md@@ -252,4 +252,10 @@ wipes + rebinds atomically when the tag changes (silent restart).Optimistic GraphQL mutations are persisted with their replay request beforebecoming visible. The exchange claims and applies them strictly in enqueueorder; a configurable callback decides whether an error remains queued or-permanently rolls back.+permanently rolls back. The exchange caps retryable server failures at ten per+mutation, excluding transport failures/timeouts. This separate budget is stored+in the existing Turso `meta` table under `mutation-server-failures:<id>` and+updated atomically with fenced deferral; absent metadata starts at zero for+existing queued writes. Commit, rollback, replacement, and clear remove it.+The storage schema/database identity and the all-attempt backoff counter stay+unchanged. Failed mutations are not retained in a DLQ.diff --git a/crates/client/cache-core/src/engine.rs b/crates/client/cache-core/src/engine.rsindex 660e562fc9..ddd13d5979 100644--- a/crates/client/cache-core/src/engine.rs+++ b/crates/client/cache-core/src/engine.rs@@ -1902,6 +1902,7 @@ impl<S: Storage> Engine<S> {claim: MutationClaimToken,next_attempt_at_ms: i64,error: String,+ server_failure: bool,) -> Result<DeferOptimisticWriteResult, EngineError<S::Error>> {self.hydrate_optimistic().await?;let layer = self@@ -1931,7 +1932,13 @@ impl<S: Storage> Engine<S> {}if !self.storage- .defer_mutation(transaction, claim, next_attempt_at_ms, error)+ .defer_mutation(+ transaction,+ claim,+ next_attempt_at_ms,+ error,+ server_failure,+ ).await.map_err(EngineError::Storage)?{diff --git a/crates/client/cache-core/src/queue.rs b/crates/client/cache-core/src/queue.rsindex baf0eaae01..54e55ea348 100644--- a/crates/client/cache-core/src/queue.rs+++ b/crates/client/cache-core/src/queue.rs@@ -37,6 +37,9 @@ pub struct StoredMutation {pub request: MutationRequest,/// Number of times this mutation has been claimed for a network attempt.pub attempt_count: u32,+ /// Retryable server failures accepted for this mutation, excluding transport failures.+ #[serde(default)]+ pub server_failure_count: u32,/// Earliest wall-clock time at which the head may be claimed again.pub next_attempt_at_ms: Option<i64>,/// Owner of the current claim, if any.@@ -57,6 +60,7 @@ impl StoredMutation {Self {request,attempt_count: 0,+ server_failure_count: 0,next_attempt_at_ms: None,lease_owner: None,lease_generation: 0,diff --git a/crates/client/cache-core/src/store.rs b/crates/client/cache-core/src/store.rsindex 8fbfa495c9..7d436c9d81 100644--- a/crates/client/cache-core/src/store.rs+++ b/crates/client/cache-core/src/store.rs@@ -207,14 +207,15 @@ pub trait Storage: MaybeSend {) -> impl Future<Output = Result<Option<ClaimedMutation>, Self::Error>> + MaybeSend;/// Retains a retryable mutation and its optimistic layer, releases its- /// lease, and records the next eligible attempt time. Returns `false`- /// when the claim is stale.+ /// lease, and records the next eligible attempt time. Server failures also+ /// increment the retry budget atomically. Returns `false` when the claim is stale.fn defer_mutation(&mut self,id: MutationId,claim: MutationClaimToken,next_attempt_at_ms: i64,error: String,+ server_failure: bool,) -> impl Future<Output = Result<bool, Self::Error>> + MaybeSend;/// Atomically writes the real response records and removes the mutation@@ -635,6 +636,7 @@ impl Storage for InMemoryStorage {claim: MutationClaimToken,next_attempt_at_ms: i64,error: String,+ server_failure: bool,) -> Result<bool, Self::Error> {let Some(queued) = self.mutations.get_mut(&id) else {return Ok(false);@@ -645,6 +647,9 @@ impl Storage for InMemoryStorage {let mutation = &mut queued.mutation;mutation.next_attempt_at_ms = Some(next_attempt_at_ms);mutation.last_error = Some(error);+ if server_failure {+ mutation.server_failure_count = mutation.server_failure_count.saturating_add(1);+ }mutation.lease_owner = None;mutation.lease_expires_at_ms = None;Ok(true)diff --git a/crates/client/cache-core/tests/mutation_queue.rs b/crates/client/cache-core/tests/mutation_queue.rsindex 6dd07bcb50..401c55dab7 100644--- a/crates/client/cache-core/tests/mutation_queue.rs+++ b/crates/client/cache-core/tests/mutation_queue.rs@@ -271,6 +271,7 @@ fn queue_claim_retry_and_settlement_are_ordered() {},200,"offline".into(),+ false,).await.unwrap()diff --git a/crates/client/cache-core/tests/optimistic.rs b/crates/client/cache-core/tests/optimistic.rsindex 964559fd6a..3fe5f0e4f2 100644--- a/crates/client/cache-core/tests/optimistic.rs+++ b/crates/client/cache-core/tests/optimistic.rs@@ -155,10 +155,11 @@ impl Storage for ClaimFailingStorage {claim: MutationClaimToken,next_attempt_at_ms: i64,error: String,+ server_failure: bool,) -> Result<bool, Self::Error> {Ok(self.inner- .defer_mutation(id, claim, next_attempt_at_ms, error)+ .defer_mutation(id, claim, next_attempt_at_ms, error, server_failure).await.unwrap())}@@ -560,7 +561,7 @@ fn enqueue_does_not_skip_a_leased_or_deferred_head() {let (_, claim) = claim_head(&mut engine, "first-runner", 10).await;if deferred {engine- .defer_optimistic_write(older, claim, 500, "offline".into())+ .defer_optimistic_write(older, claim, 500, "offline".into(), false).await.unwrap();}@@ -826,7 +827,7 @@ fn retryable_failure_keeps_optimistic_layer_and_blocks_later_mutations() {let (_, claim) = claim_head(&mut engine, "runner", 10).await;engine- .defer_optimistic_write(first, claim, 100, "offline".into())+ .defer_optimistic_write(first, claim, 100, "offline".into(), false).await.unwrap();assert!(@@ -1085,6 +1086,7 @@ fn active_uuid_replacement_is_superseded_and_failed_attempt_is_discarded() {},500,"superseded".into(),+ false,).await.unwrap();diff --git a/crates/client/cache-core/tests/query_inspection.rs b/crates/client/cache-core/tests/query_inspection.rsindex ec3368fd74..9b328cb456 100644--- a/crates/client/cache-core/tests/query_inspection.rs+++ b/crates/client/cache-core/tests/query_inspection.rs@@ -147,9 +147,10 @@ impl Storage for OwnerOnlyStorage {claim: MutationClaimToken,next_attempt_at_ms: i64,error: String,+ server_failure: bool,) -> Result<bool, Self::Error> {self.0- .defer_mutation(id, claim, next_attempt_at_ms, error)+ .defer_mutation(id, claim, next_attempt_at_ms, error, server_failure).await}diff --git a/crates/client/cache-turso/src/storage.rs b/crates/client/cache-turso/src/storage.rsindex c426fb49c0..17d9af046a 100644--- a/crates/client/cache-turso/src/storage.rs+++ b/crates/client/cache-turso/src/storage.rs@@ -133,8 +133,8 @@ const SORT_FACT_INSERT: &str ="INSERT INTO sort_facts (document_id, attribute, value) VALUES (?1, ?2, ?3)";const QUEUE_INSERT: &str = "INSERT INTO mutation_queue (uuid, superseded, query, operation_name, variables_json, identity, attempt_count, next_attempt_at_ms, lease_owner, lease_generation, lease_expires_at_ms, last_error, created_at_ms) VALUES (?1, 0, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)";const LAYER_INSERT: &str = "INSERT INTO optimistic_layers (mutation_id, optimistic_data_json, normalized_updates) VALUES (?1, ?2, ?3)";-const QUEUE_SELECT: &str = "SELECT m.id, m.uuid, m.superseded, m.query, m.operation_name, m.variables_json, m.identity, m.attempt_count, m.next_attempt_at_ms, m.lease_owner, m.lease_generation, m.lease_expires_at_ms, m.last_error, m.created_at_ms, o.optimistic_data_json, o.normalized_updates FROM mutation_queue AS m LEFT JOIN optimistic_layers AS o ON o.mutation_id = m.id ORDER BY m.id ASC";-const QUEUE_HEAD_SELECT: &str = "SELECT m.id, m.uuid, m.superseded, m.query, m.operation_name, m.variables_json, m.identity, m.attempt_count, m.next_attempt_at_ms, m.lease_owner, m.lease_generation, m.lease_expires_at_ms, m.last_error, m.created_at_ms, o.optimistic_data_json, o.normalized_updates FROM mutation_queue AS m LEFT JOIN optimistic_layers AS o ON o.mutation_id = m.id ORDER BY m.id ASC LIMIT 1";+const QUEUE_SELECT: &str = "SELECT m.id, m.uuid, m.superseded, m.query, m.operation_name, m.variables_json, m.identity, m.attempt_count, m.next_attempt_at_ms, m.lease_owner, m.lease_generation, m.lease_expires_at_ms, m.last_error, m.created_at_ms, o.optimistic_data_json, o.normalized_updates, r.value FROM mutation_queue AS m LEFT JOIN optimistic_layers AS o ON o.mutation_id = m.id LEFT JOIN meta AS r ON r.key = 'mutation-server-failures:' || m.id ORDER BY m.id ASC";+const QUEUE_HEAD_SELECT: &str = "SELECT m.id, m.uuid, m.superseded, m.query, m.operation_name, m.variables_json, m.identity, m.attempt_count, m.next_attempt_at_ms, m.lease_owner, m.lease_generation, m.lease_expires_at_ms, m.last_error, m.created_at_ms, o.optimistic_data_json, o.normalized_updates, r.value FROM mutation_queue AS m LEFT JOIN optimistic_layers AS o ON o.mutation_id = m.id LEFT JOIN meta AS r ON r.key = 'mutation-server-failures:' || m.id ORDER BY m.id ASC LIMIT 1";const ORPHAN_LAYER_SELECT: &str = "SELECT o.mutation_id FROM optimistic_layers AS o LEFT JOIN mutation_queue AS m ON m.id = o.mutation_id WHERE m.id IS NULL LIMIT 1";const ANY_LAYER_SELECT: &str = "SELECT mutation_id FROM optimistic_layers LIMIT 1";const CLAIM_SELECT: &str = "SELECT lease_owner, lease_generation FROM mutation_queue WHERE id = ?1";@@ -1041,6 +1041,7 @@ impl Storage for TursoStorage {)?,1,)?;+ mutation_retry::remove(&connection, mutation_id_to_sql(existing)?)?;MutationUpsertKind::ReplacedPending {removed_id: existing,}@@ -1054,6 +1055,11 @@ impl Storage for TursoStorage {1,)?;let id = mutation_id_from_row(connection.last_insert_rowid())?;+ mutation_retry::save(+ &connection,+ mutation_id_to_sql(id)?,+ entry.mutation.server_failure_count,+ )?;self.fault_after(TestFaultSite::Enqueue, 1)?;require_changed(driver::execute(@@ -1227,6 +1233,7 @@ impl Storage for TursoStorage {claim: MutationClaimToken,next_attempt_at_ms: i64,error: String,+ server_failure: bool,) -> Result<bool, Self::Error> {self.require_healthy()?;let result = (|| {@@ -1245,7 +1252,11 @@ impl Storage for TursoStorage {text(&error),],)?;- changed_to_bool(changed)+ let accepted = changed_to_bool(changed)?;+ if accepted && server_failure {+ mutation_retry::increment(&connection, id)?;+ }+ Ok(accepted)})})();self.latch_result(result)@@ -1306,6 +1317,7 @@ impl Storage for TursoStorage {)?,1,)?;+ mutation_retry::remove(&connection, sql_id)?;Ok(true)})})();@@ -1361,6 +1373,7 @@ impl Storage for TursoStorage {)?,1,)?;+ mutation_retry::remove(&connection, sql_id)?;write_shadow_reconciliation(&connection, reconciliation, |index| {self.fault_after(TestFaultSite::Complete, entries.len() + index)})?;@@ -1393,6 +1406,7 @@ impl Storage for TursoStorage {)?,1,)?;+ mutation_retry::remove(&connection, sql_id)?;self.fault_after(TestFaultSite::Discard, 0)?;Ok(true)})@@ -1427,6 +1441,7 @@ impl Storage for TursoStorage {)?,1,)?;+ mutation_retry::remove(&connection, sql_id)?;write_shadow_reconciliation(&connection, reconciliation, |index| {self.fault_after(TestFaultSite::Discard, index)})?;@@ -1444,6 +1459,7 @@ impl Storage for TursoStorage {driver::execute(&connection, "DELETE FROM optimistic_layers", Vec::new())?;self.fault_after(TestFaultSite::Clear, 0)?;driver::execute(&connection, "DELETE FROM mutation_queue", Vec::new())?;+ mutation_retry::clear(&connection)?;self.fault_after(TestFaultSite::Clear, 1)?;driver::execute(&connection, "DELETE FROM search_documents", Vec::new())?;driver::execute(&connection, "DELETE FROM index_documents", Vec::new())?;@@ -4678,7 +4694,7 @@ struct ParsedQueueRow {}fn parse_queue_row(row: &[Value]) -> Result<ParsedQueueRow, TursoStorageError> {- if row.len() != 16 {+ if row.len() != 17 {return Err(invariant());}let optimistic = match (&row[14], &row[15]) {@@ -4702,6 +4718,7 @@ fn parse_queue_row(row: &[Value]) -> Result<ParsedQueueRow, TursoStorageError> {identity: nullable_text(row, 6)?,},attempt_count: u32::try_from(required_i64(row, 7)?).map_err(|_| invariant())?,+ server_failure_count: mutation_retry::parse(nullable_text(row, 16)?)?,next_attempt_at_ms: nullable_i64(row, 8)?,lease_owner: nullable_text(row, 9)?,lease_generation: u64::try_from(required_i64(row, 10)?).map_err(|_| invariant())?,@@ -4953,6 +4970,7 @@ mod alternatives;mod bounded_selection;mod conjunction;mod integrity;+mod mutation_retry;mod page_retention;mod record_batch;diff --git a/crates/client/cache-turso/src/storage/mutation_retry.rs b/crates/client/cache-turso/src/storage/mutation_retry.rsnew file mode 100644index 0000000000..2f07ccc333--- /dev/null+++ b/crates/client/cache-turso/src/storage/mutation_retry.rs@@ -0,0 +1,63 @@+//! Pending-mutation retry budgets in the existing metadata table. Keeping the+//! frozen storage schema and database identity unchanged preserves queued writes+//! across upgrades. All writes here run inside the queue's fenced transaction.++use super::{Connection, TursoStorageError, driver, invariant, nullable_text, text};+use std::sync::Arc;++const KEY_PREFIX: &str = "mutation-server-failures:";++pub(super) fn parse(value: Option<String>) -> Result<u32, TursoStorageError> {+ value+ .map(|value| value.parse().map_err(|_| invariant()))+ .transpose()+ .map(Option::unwrap_or_default)+}++pub(super) fn save(+ connection: &Arc<Connection>,+ id: i64,+ count: u32,+) -> Result<(), TursoStorageError> {+ if count == 0 {+ return Ok(());+ }+ driver::execute(+ connection,+ "INSERT INTO meta (key, value) VALUES (?1, ?2) ON CONFLICT (key) DO UPDATE SET value = excluded.value",+ vec![text(&format!("{KEY_PREFIX}{id}")), text(&count.to_string())],+ )?;+ Ok(())+}++pub(super) fn increment(connection: &Arc<Connection>, id: i64) -> Result<(), TursoStorageError> {+ let rows = driver::query(+ connection,+ "SELECT value FROM meta WHERE key = ?1",+ vec![text(&format!("{KEY_PREFIX}{id}"))],+ )?;+ let count = match rows.as_slice() {+ [] => 0,+ [row] => parse(nullable_text(row, 0)?)?,+ _ => return Err(invariant()),+ };+ save(connection, id, count.saturating_add(1))+}++pub(super) fn remove(connection: &Arc<Connection>, id: i64) -> Result<(), TursoStorageError> {+ driver::execute(+ connection,+ "DELETE FROM meta WHERE key = ?1",+ vec![text(&format!("{KEY_PREFIX}{id}"))],+ )?;+ Ok(())+}++pub(super) fn clear(connection: &Arc<Connection>) -> Result<(), TursoStorageError> {+ driver::execute(+ connection,+ "DELETE FROM meta WHERE key GLOB ?1",+ vec![text(&format!("{KEY_PREFIX}*"))],+ )?;+ Ok(())+}diff --git a/crates/client/cache-turso/src/storage/test.rs b/crates/client/cache-turso/src/storage/test.rsindex 1e289322f2..638b12898c 100644--- a/crates/client/cache-turso/src/storage/test.rs+++ b/crates/client/cache-turso/src/storage/test.rs@@ -6,6 +6,7 @@ mod engine_writes;mod fact_lookup_cost;mod filter_scope_cost;mod filter_scope_semantics;+mod mutation_retry;mod page_retention;mod predicate_cost;mod projection_writes;@@ -168,7 +169,7 @@ async fn expect_every_storage_method_latched();expect_reset_reason(storage- .defer_mutation(1, token("blocked", 1), 2, "blocked".into())+ .defer_mutation(1, token("blocked", 1), 2, "blocked".into(), false).await,expected,);diff --git a/crates/client/cache-turso/src/storage/test/mutation_retry.rs b/crates/client/cache-turso/src/storage/test/mutation_retry.rsnew file mode 100644index 0000000000..08b57a2dc3--- /dev/null+++ b/crates/client/cache-turso/src/storage/test/mutation_retry.rs@@ -0,0 +1,134 @@+use super::*;++fn budget_rows(storage: &TursoStorage) -> i64 {+ raw_scalar(+ storage,+ "SELECT COUNT(*) FROM meta WHERE key GLOB 'mutation-server-failures:*'",+ )+}++#[test]+fn retry_budget_is_removed_with_its_pending_mutation() {+ block_on(async {+ for mode in ["commit", "rollback", "replace", "clear"] {+ let database = TursoMemoryDatabase::new(format!("retry-cleanup-{mode}.db"));+ let mut storage = database.open("retry-cleanup").unwrap();+ let mut entry = queued("Pending");+ let uuid = entry.uuid;+ entry.mutation.server_failure_count = 9;+ let id = storage.enqueue_mutation(entry).await.unwrap();+ assert_eq!(budget_rows(&storage), 1);+ match mode {+ "replace" => {+ let mut replacement = queued("Replacement");+ replacement.uuid = uuid;+ storage.enqueue_mutation(replacement).await.unwrap();+ assert_eq!(+ storage.load_mutation_queue().await.unwrap()[0]+ .mutation+ .server_failure_count,+ 0+ );+ }+ "clear" => storage.clear().await.unwrap(),+ _ => {+ let claim = storage+ .claim_next_mutation(MutationClaimRequest {+ owner: "runner".into(),+ now_ms: 1,+ lease_expires_at_ms: 2,+ })+ .await+ .unwrap()+ .unwrap();+ assert_eq!(claim.queued.mutation.server_failure_count, 9);+ let claim = token("runner", claim.lease_generation);+ if mode == "commit" {+ assert!(+ storage+ .complete_mutation(id, claim, Vec::new())+ .await+ .unwrap()+ );+ } else {+ assert!(storage.discard_mutation(id, claim).await.unwrap());+ }+ }+ }+ assert_eq!(budget_rows(&storage), 0, "{mode}");+ storage.try_close().unwrap();+ }+ });+}++#[test]+fn retry_budget_survives_file_reopen_without_a_schema_change() {+ block_on(async {+ let directory = tempfile::tempdir().unwrap();+ let path = directory.path().join("retry.db");+ let database = TursoFileDatabase::new(&path).unwrap();+ let mut storage = database.open("retry").unwrap();+ let id = storage+ .enqueue_mutation(queued("LegacyPending"))+ .await+ .unwrap();+ assert_eq!(budget_rows(&storage), 0);+ let claim = storage+ .claim_next_mutation(MutationClaimRequest {+ owner: "old-tab".into(),+ now_ms: 1,+ lease_expires_at_ms: 2,+ })+ .await+ .unwrap()+ .unwrap();+ assert_eq!(claim.queued.mutation.server_failure_count, 0);+ assert!(+ storage+ .defer_mutation(+ id,+ token("old-tab", claim.lease_generation),+ 2,+ "server failed".into(),+ true+ )+ .await+ .unwrap()+ );+ storage.try_close().unwrap();+ drop(database);+ let database = TursoFileDatabase::new(path).unwrap();+ let mut storage = database.open("retry").unwrap();+ let claim = storage+ .claim_next_mutation(MutationClaimRequest {+ owner: "new-tab".into(),+ now_ms: 2,+ lease_expires_at_ms: 3,+ })+ .await+ .unwrap()+ .unwrap();+ assert_eq!(claim.queued.id, id);+ assert_eq!(claim.queued.mutation.attempt_count, 2);+ assert_eq!(claim.queued.mutation.server_failure_count, 1);+ assert!(+ storage+ .defer_mutation(+ id,+ token("new-tab", claim.lease_generation),+ 3,+ "offline".into(),+ false+ )+ .await+ .unwrap()+ );+ assert_eq!(+ storage.load_mutation_queue().await.unwrap()[0]+ .mutation+ .server_failure_count,+ 1+ );+ storage.try_close().unwrap();+ });+}diff --git a/crates/client/cache-turso/tests/draft_identity.rs b/crates/client/cache-turso/tests/draft_identity.rsindex 3751e12c0c..079a727a9a 100644--- a/crates/client/cache-turso/tests/draft_identity.rs+++ b/crates/client/cache-turso/tests/draft_identity.rs@@ -168,7 +168,7 @@ fn disk_restart_rebases_newer_edits_and_preserves_uncertain_save_before_discard(let first_claim = claim(&mut engine, first, 1).await;assert!(matches!(engine- .defer_optimistic_write(first, first_claim, 20, "uncertain response".into())+ .defer_optimistic_write(first, first_claim, 20, "uncertain response".into(), false).await.unwrap(),DeferOptimisticWriteResult::Deferred@@ -219,7 +219,7 @@ fn disk_restart_rebases_newer_edits_and_preserves_uncertain_save_before_discard(assert!(read(&mut engine, "local-draft").await.is_empty());assert!(matches!(engine- .defer_optimistic_write(newer, newer_claim, 40, "uncertain edit".into())+ .defer_optimistic_write(newer, newer_claim, 40, "uncertain edit".into(), false).await.unwrap(),DeferOptimisticWriteResult::Deferreddiff --git a/crates/client/cache-turso/tests/engine_turso.rs b/crates/client/cache-turso/tests/engine_turso.rsindex 8ca065f2b0..2e6c0ed19f 100644--- a/crates/client/cache-turso/tests/engine_turso.rs+++ b/crates/client/cache-turso/tests/engine_turso.rs@@ -242,6 +242,7 @@ fn optimistic_hydration_retry_complete_and_reopen_run_over_turso() {},200,"offline".into(),+ false,).await.unwrap();diff --git a/crates/client/cache-turso/tests/shared_conformance.rs b/crates/client/cache-turso/tests/shared_conformance.rsindex cbef13471e..ca87868800 100644--- a/crates/client/cache-turso/tests/shared_conformance.rs+++ b/crates/client/cache-turso/tests/shared_conformance.rs@@ -106,6 +106,7 @@ fn fully_populated_queued() -> NewQueuedMutation {identity: Some("identity-witness".into()),},attempt_count: 7,+ server_failure_count: 3,next_attempt_at_ms: Some(-2),lease_owner: Some("expired-owner".into()),lease_generation: 11,@@ -343,7 +344,7 @@ async fn queue_contract<S: Storage>(storage: &mut S) {let absent = token("absent", 1);assert!(!storage- .defer_mutation(999, absent.clone(), 2, "absent".into())+ .defer_mutation(999, absent.clone(), 2, "absent".into(), true).await.unwrap());@@ -397,6 +398,7 @@ async fn queue_contract<S: Storage>(storage: &mut S) {token("runner-a", first_claim.lease_generation),20,"stale".into(),+ true,).await.unwrap()@@ -414,6 +416,7 @@ async fn queue_contract<S: Storage>(storage: &mut S) {token("runner-b", expired_reclaim.lease_generation),20,"retry".into(),+ true,).await.unwrap()@@ -471,6 +474,7 @@ async fn queue_contract<S: Storage>(storage: &mut S) {token("runner-d", second_claim.lease_generation),30,"second retry".into(),+ false,).await.unwrap()@@ -527,6 +531,58 @@ async fn queue_contract<S: Storage>(storage: &mut S) {);}+async fn retry_budget_contract<F: BackendFactory>(+ factory: &mut F,+ mut storage: F::Backend,+) -> F::Backend {+ let id = storage+ .enqueue_mutation(queued("RetryBudget"))+ .await+ .unwrap();+ for (index, (server_failure, expected_count)) in+ [(true, 1), (false, 1), (true, 2)].into_iter().enumerate()+ {+ let now = index as i64;+ let claimed = storage+ .claim_next_mutation(claim_request("runner", now, now + 1))+ .await+ .unwrap()+ .unwrap();+ assert_eq!(claimed.queued.id, id);+ assert_eq!(claimed.queued.mutation.attempt_count, index as u32 + 1);+ let claim = token("runner", claimed.lease_generation);+ assert!(+ storage+ .defer_mutation(id, claim.clone(), now + 1, "retry".into(), server_failure)+ .await+ .unwrap()+ );+ // A duplicate result must not charge the same attempt twice.+ assert!(+ !storage+ .defer_mutation(id, claim, now + 1, "duplicate".into(), true)+ .await+ .unwrap()+ );+ storage = factory.reopen(storage);+ let queue = storage.load_mutation_queue().await.unwrap();+ assert_eq!(queue[0].mutation.server_failure_count, expected_count);+ }+ let claimed = storage+ .claim_next_mutation(claim_request("next-tab", 3, 4))+ .await+ .unwrap()+ .unwrap();+ assert_eq!(claimed.queued.mutation.server_failure_count, 2);+ assert!(+ storage+ .discard_mutation(id, token("next-tab", claimed.lease_generation))+ .await+ .unwrap()+ );+ storage+}+async fn clear_contract<S: Storage>(storage: &mut S) {let before_clear = storage.enqueue_mutation(queued("Clear")).await.unwrap();storage.clear().await.unwrap();@@ -570,6 +626,7 @@ async fn common_contract<F: BackendFactory>(search_projection_contract(&mut storage).await;let mut storage = reopen_contract(factory, storage).await;queue_contract(&mut storage).await;+ let mut storage = retry_budget_contract(factory, storage).await;clear_contract(&mut storage).await;storage}diff --git a/crates/client/cache-turso/tests/storage_conformance.rs b/crates/client/cache-turso/tests/storage_conformance.rsindex 593e35a9d6..2b99a8d5fa 100644--- a/crates/client/cache-turso/tests/storage_conformance.rs+++ b/crates/client/cache-turso/tests/storage_conformance.rs@@ -37,6 +37,7 @@ fn queued(label: &str, created_at_ms: i64) -> NewQueuedMutation {identity: Some("identity-witness".into()),},attempt_count: 7,+ server_failure_count: 3,next_attempt_at_ms: None,lease_owner: None,lease_generation: 11,@@ -307,13 +308,13 @@ fn strict_head_retries_leases_and_fences_conform() {assert_eq!(reclaimed.lease_generation, 2);assert!(!storage- .defer_mutation(first, claim("runner-a", 1), 300, "stale".into())+ .defer_mutation(first, claim("runner-a", 1), 300, "stale".into(), true).await.unwrap());assert!(storage- .defer_mutation(first, claim("runner-b", 2), 300, "offline".into())+ .defer_mutation(first, claim("runner-b", 2), 300, "offline".into(), false).await.unwrap());diff --git a/crates/client/cache-wasm/src/shell.rs b/crates/client/cache-wasm/src/shell.rsindex 45d9343497..07e471bad1 100644--- a/crates/client/cache-wasm/src/shell.rs+++ b/crates/client/cache-wasm/src/shell.rs@@ -246,6 +246,7 @@ struct JsClaimedMutation {variables: serde_json::Value,identity: Option<String>,attempt_count: u32,+ server_failure_count: u32,}impl TryFrom<ClaimedMutation> for JsClaimedMutation {@@ -265,6 +266,7 @@ impl TryFrom<ClaimedMutation> for JsClaimedMutation {variables: serde_json::from_str(&request.variables_json).map_err(err_js)?,identity: request.identity,attempt_count: claimed.queued.mutation.attempt_count,+ server_failure_count: claimed.queued.mutation.server_failure_count,})}}@@ -1863,6 +1865,7 @@ impl CacheEngine {lease_generation: String,next_attempt_at_ms: f64,error: String,+ server_failure: bool,) -> js_sys::Promise {let state = self.state.clone();let ops = self.ops.clone();@@ -1877,7 +1880,13 @@ impl CacheEngine {let next_attempt_at_ms = parse_timestamp(next_attempt_at_ms, "next attempt timestamp")?;let result = state.engine_mut()?- .defer_optimistic_write(transaction, claim, next_attempt_at_ms, error)+ .defer_optimistic_write(+ transaction,+ claim,+ next_attempt_at_ms,+ error,+ server_failure,+ ).await;let result = state.engine_result(result)?;let result = match result {diff --git a/crates/client/cache-wasm/src/shell/test.rs b/crates/client/cache-wasm/src/shell/test.rsindex 091e50df90..233134b25a 100644--- a/crates/client/cache-wasm/src/shell/test.rs+++ b/crates/client/cache-wasm/src/shell/test.rs@@ -2210,6 +2210,7 @@ async fn every_method_rejects_after_consuming_close() {"1".into(),2.0,"closed".into(),+ false,)).await;assert_closed(engine.commit_optimistic_write(diff --git a/crates/client/cache-wasm/src/shell/test_storage.rs b/crates/client/cache-wasm/src/shell/test_storage.rsindex 63ac7a119c..4f1c3cea14 100644--- a/crates/client/cache-wasm/src/shell/test_storage.rs+++ b/crates/client/cache-wasm/src/shell/test_storage.rs@@ -156,9 +156,10 @@ impl Storage for BrowserStorage {claim: MutationClaimToken,next_attempt_at_ms: i64,error: String,+ server_failure: bool,) -> Result<bool, Self::Error> {self.inner- .defer_mutation(id, claim, next_attempt_at_ms, error)+ .defer_mutation(id, claim, next_attempt_at_ms, error, server_failure).await}