Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

Seanaye/feat/purge queue (#7635)

Sean Ayea6d16828fc05
diff --git a/apps/web/docs/graphql-normalized-cache-plan.md b/apps/web/docs/graphql-normalized-cache-plan.md
index 9251cf12eb..0c936f8d89 100644
--- a/apps/web/docs/graphql-normalized-cache-plan.md
+++ b/apps/web/docs/graphql-normalized-cache-plan.md
@@ -100,8 +100,15 @@ entire cache in browser memory. With 10s of thousands of cached objects
13. **Durable optimistic mutations** — optimistic GraphQL mutations are
persisted with their replay request, restored across restarts, and applied
strictly in enqueue order. Retryability is decided by an exchange callback;
- retryable failures retain their optimistic layer. Each queued network
- attempt has a one-minute timeout, comfortably inside its five-minute lease.
+ retryable failures retain their optimistic layer. Ten retryable server
+ failures (GraphQL errors or HTTP 5xx responses) permanently fail the mutation
+ through ordinary rollback so the next queued mutation can run. Transport
+ failures/timeouts do not consume or reset that budget. The separate server
+ failure counter survives reloads and tab handoff in pending-mutation metadata;
+ `attempt_count` still counts every claim for backoff and coalescing safety.
+ Settlement removes the pending mutation and counter; there is no retained DLQ.
+ Each queued network attempt has a one-minute timeout, comfortably inside its
+ five-minute lease.
### Open questions
diff --git a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts
index 016120b5ae..b2a02bc03b 100644
--- a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts
+++ b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts
@@ -12,7 +12,9 @@ import { shouldRetryGraphqlMutation } from '@service-storage/graphql-mutation-re
import {
type Client,
CombinedError,
+ createClient,
createRequest,
+ fetchExchange,
gql,
makeOperation,
type Operation,
@@ -49,6 +51,7 @@ import {
normalizedCacheResultMetadata,
} from './normalized-cache-exchange';
import {
+ executeOptimisticMutation,
optimisticContextOf,
optimisticMutationDispositionOf,
} from './optimistic';
@@ -205,7 +208,8 @@ type FakeHost = CacheHost & {
teardowns: number[];
scriptRead: (result: ReadResult) => void;
seedQueued: (
- args: Parameters<CacheHost['enqueueOptimisticMutation']>[0]
+ args: Parameters<CacheHost['enqueueOptimisticMutation']>[0],
+ serverFailureCount?: number
) => void;
pushAffected: (opKeys: number[]) => void;
pushGeneration: (change: CacheGenerationChange) => void;
@@ -221,6 +225,7 @@ function makeFakeHost(): FakeHost {
transactionId: string;
args: Parameters<CacheHost['enqueueOptimisticMutation']>[0];
attemptCount: number;
+ serverFailureCount: number;
leased: boolean;
nextAttemptAtMs?: number;
}> = [];
@@ -248,6 +253,7 @@ function makeFakeHost(): FakeHost {
operationName: head.args.operationName,
variables: head.args.variables ?? {},
attemptCount: head.attemptCount,
+ serverFailureCount: head.serverFailureCount,
};
}
@@ -266,11 +272,12 @@ function makeFakeHost(): FakeHost {
scriptRead: (r) => {
readResult = r;
},
- seedQueued: (args) => {
+ seedQueued: (args, serverFailureCount = 0) => {
queue.push({
transactionId: `restored-${queue.length + 1}`,
args,
attemptCount: 0,
+ serverFailureCount,
leased: false,
});
},
@@ -341,7 +348,13 @@ function makeFakeHost(): FakeHost {
linkPatches: args.linkPatches,
});
const transactionId = `txn-${host.begins.length}`;
- queue.push({ transactionId, args, attemptCount: 0, leased: false });
+ queue.push({
+ transactionId,
+ args,
+ attemptCount: 0,
+ serverFailureCount: 0,
+ leased: false,
+ });
const mutation = claimQueueHead(claim.nowMs);
return {
transactionId,
@@ -372,13 +385,15 @@ function makeFakeHost(): FakeHost {
transactionId,
_claim: MutationClaim,
nextAttemptAtMs,
- error
+ error,
+ serverFailure = false
) {
host.defers.push({ transactionId, error });
const head = queue[0];
if (head?.transactionId === transactionId) {
head.leased = false;
head.nextAttemptAtMs = nextAttemptAtMs;
+ if (serverFailure) head.serverFailureCount += 1;
}
return { kind: 'deferred' };
},
@@ -4002,6 +4017,285 @@ describe('normalizedCacheExchange', () => {
});
});
+ it.each([
+ { status: 500, data: null, failures: 0 },
+ { status: 503, data: { from: 'server' }, failures: 0 },
+ { status: 500, data: null, failures: 9 },
+ { status: 503, data: { from: 'server' }, failures: 9 },
+ ])(
+ 'handles HTTP $status without GraphQL errors after $failures server failures',
+ async ({ status, data, failures }) => {
+ vi.useFakeTimers();
+ try {
+ if (failures > 0) {
+ host.seedQueued(
+ {
+ uuid: crypto.randomUUID(),
+ query: stringifyDocument(MUTATION),
+ data: optimistic,
+ },
+ failures
+ );
+ }
+ const response = new Response(JSON.stringify({ data }), {
+ status,
+ headers: { 'content-type': 'application/json' },
+ });
+ const shouldRetryMutation = vi.fn(shouldRetryGraphqlMutation);
+ const rollback = vi.spyOn(host, 'rollbackOptimisticWrite');
+ const defer = vi.spyOn(host, 'deferOptimisticWrite');
+ const client = createClient({
+ url: 'http://test/graphql',
+ fetch: vi.fn(async () => response),
+ exchanges: [
+ normalizedCacheExchange(host, { shouldRetryMutation }),
+ fetchExchange,
+ ],
+ });
+ const pending =
+ failures === 0
+ ? executeOptimisticMutation(
+ client,
+ MUTATION,
+ { input: {} },
+ optimistic,
+ { uuid: crypto.randomUUID() }
+ ).toPromise()
+ : undefined;
+ await vi.advanceTimersByTimeAsync(10);
+ expect(shouldRetryMutation).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({ response, graphQLErrors: [] })
+ );
+ expect(host.commits).toHaveLength(0);
+ if (failures === 0) {
+ expect(defer).toHaveBeenCalledExactlyOnceWith(
+ 'txn-1',
+ expect.any(Object),
+ expect.any(Number),
+ expect.stringContaining(String(status)),
+ true
+ );
+ expect(host.rollbacks).toHaveLength(0);
+ expect(optimisticMutationDispositionOf((await pending)!)).toEqual({
+ kind: 'queued',
+ transactionId: 'txn-1',
+ });
+ } else {
+ expect(host.defers).toHaveLength(0);
+ expect(rollback).toHaveBeenCalledExactlyOnceWith(
+ 'restored-1',
+ expect.any(Object),
+ expect.stringContaining('10 server failures'),
+ 'MUTATION_RETRY_EXHAUSTED'
+ );
+ }
+ } finally {
+ vi.clearAllTimers();
+ vi.useRealTimers();
+ }
+ }
+ );
+
+ it.each([false, true])(
+ 'preserves success and explicit GraphQL rejection with fetchExchange (rejected=%s)',
+ async (rejected) => {
+ vi.useFakeTimers();
+ try {
+ const response = new Response(
+ JSON.stringify(
+ rejected
+ ? {
+ errors: [
+ { message: 'rejected', extensions: { code: 'INVALID' } },
+ ],
+ }
+ : { data: optimistic }
+ ),
+ {
+ status: rejected ? 503 : 200,
+ headers: { 'content-type': 'application/json' },
+ }
+ );
+ const client = createClient({
+ url: 'http://test/graphql',
+ fetch: vi.fn(async () => response),
+ exchanges: [
+ normalizedCacheExchange(host, {
+ shouldRetryMutation: shouldRetryGraphqlMutation,
+ }),
+ fetchExchange,
+ ],
+ });
+ const pending = executeOptimisticMutation(
+ client,
+ MUTATION,
+ { input: {} },
+ optimistic,
+ { uuid: crypto.randomUUID() }
+ ).toPromise();
+ await vi.advanceTimersByTimeAsync(10);
+ const result = await pending;
+ expect(host.defers).toHaveLength(0);
+ expect(optimisticMutationDispositionOf(result)?.kind).toBe(
+ rejected ? 'permanently-failed' : 'committed'
+ );
+ if (rejected) {
+ expect(result.error?.graphQLErrors[0]?.extensions.code).toBe(
+ 'INVALID'
+ );
+ expect(result.error?.response).toBe(response);
+ }
+ } finally {
+ vi.clearAllTimers();
+ vi.useRealTimers();
+ }
+ }
+ );
+
+ it.each(['graphql', 'http-500', 'http-503'])(
+ 'permanently fails the tenth %s failure and advances the queue',
+ async (kind) => {
+ vi.useFakeTimers();
+ try {
+ const error =
+ kind === 'graphql'
+ ? new CombinedError({
+ graphQLErrors: [
+ {
+ message: 'persistent failure',
+ extensions: { retryable: true },
+ },
+ ],
+ })
+ : new CombinedError({
+ networkError: new Error('server unavailable'),
+ response: new Response(null, {
+ status: kind === 'http-500' ? 500 : 503,
+ }),
+ });
+ const rollback = vi.spyOn(host, 'rollbackOptimisticWrite');
+ let attempts = 0;
+ const { ops, results, forwarded } = harness(
+ host,
+ () =>
+ ++attempts <= 10
+ ? { error, data: undefined }
+ : { data: optimistic },
+ { shouldRetryMutation: shouldRetryGraphqlMutation }
+ );
+ ops.next(makeMutationOp(1, optimistic));
+ ops.next(makeMutationOp(2, optimistic));
+ await vi.advanceTimersByTimeAsync(0);
+ for (let attempt = 1; attempt < 10; attempt += 1) {
+ expect(forwarded).toHaveLength(attempt);
+ expect(host.rollbacks).toHaveLength(0);
+ await vi.advanceTimersByTimeAsync(
+ Math.min(1_000 * 2 ** (attempt - 1), 60_000)
+ );
+ }
+ await vi.advanceTimersByTimeAsync(1);
+ expect(host.defers).toHaveLength(9);
+ expect(rollback).toHaveBeenCalledExactlyOnceWith(
+ 'txn-1',
+ expect.any(Object),
+ expect.stringContaining('10 server failures'),
+ 'MUTATION_RETRY_EXHAUSTED'
+ );
+ expect(host.commits.map((entry) => entry.transactionId)).toEqual([
+ 'txn-2',
+ ]);
+ expect(forwarded).toHaveLength(11);
+ expect(
+ results.some(
+ (result) =>
+ optimisticMutationDispositionOf(result)?.kind ===
+ 'permanently-failed'
+ )
+ ).toBe(true);
+ } finally {
+ vi.clearAllTimers();
+ vi.useRealTimers();
+ }
+ }
+ );
+
+ it('network failures neither consume nor reset the server failure budget', async () => {
+ vi.useFakeTimers();
+ try {
+ const server = new CombinedError({
+ graphQLErrors: [
+ { message: 'server failed', extensions: { retryable: true } },
+ ],
+ });
+ const network = new CombinedError({
+ networkError: new DOMException('request timed out', 'TimeoutError'),
+ });
+ let attempts = 0;
+ const { ops } = harness(
+ host,
+ () => {
+ attempts += 1;
+ // Transport failures before the first server error cannot consume
+ // the budget; transport failures after nine cannot reset it either.
+ return {
+ error:
+ (attempts >= 13 && attempts <= 21) || attempts === 34
+ ? server
+ : network,
+ data: undefined,
+ };
+ },
+ { shouldRetryMutation: shouldRetryGraphqlMutation }
+ );
+ ops.next(makeMutationOp(1, optimistic));
+ await vi.advanceTimersByTimeAsync(0);
+ for (let attempt = 1; attempt < 34; attempt += 1) {
+ expect(host.rollbacks).toHaveLength(0);
+ await vi.advanceTimersByTimeAsync(
+ Math.min(1_000 * 2 ** (attempt - 1), 60_000)
+ );
+ }
+ expect(attempts).toBe(34);
+ expect(host.defers).toHaveLength(33);
+ expect(host.rollbacks).toEqual(['txn-1']);
+ } finally {
+ vi.clearAllTimers();
+ vi.useRealTimers();
+ }
+ });
+
+ it('uses the persisted budget on replay and revalidates after exhaustion', async () => {
+ host.seedQueued(
+ {
+ uuid: crypto.randomUUID(),
+ query: stringifyDocument(MUTATION),
+ data: optimistic,
+ revalidations: [
+ { query: stringifyDocument(QUERY), variablesJson: '{}' },
+ ],
+ },
+ 9
+ );
+ const rollback = vi.spyOn(host, 'rollbackOptimisticWrite');
+ const error = new CombinedError({
+ graphQLErrors: [
+ { message: 'still failing', extensions: { retryable: true } },
+ ],
+ });
+ const { client } = harness(host, () => ({ error, data: undefined }), {
+ shouldRetryMutation: shouldRetryGraphqlMutation,
+ });
+ await tick();
+ expect(host.defers).toHaveLength(0);
+ expect(rollback).toHaveBeenCalledExactlyOnceWith(
+ 'restored-1',
+ expect.any(Object),
+ expect.stringContaining('10 server failures'),
+ 'MUTATION_RETRY_EXHAUSTED'
+ );
+ expect(client.query).toHaveBeenCalledOnce();
+ });
+
it('retries a draft response failure with the same handles and commits after recovery', async () => {
vi.useFakeTimers();
try {
diff --git a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts
index 9205342eaa..cd20d27015 100644
--- a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts
+++ b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts
@@ -23,8 +23,9 @@
* Mutations:
* - With an optimistic response (see `executeOptimisticMutation`): the
* mutation and layer are durably queued before the ordered runner forwards
- * it. Retryable failures retain optimism for background replay; permanent
- * failures roll back. A caller whose operation is blocked behind the queue
+ * it. Retryable failures retain optimism for background replay, but the tenth
+ * server failure permanently rolls back; transport failures are not counted.
+ * Other permanent failures roll back immediately. A caller blocked behind the queue
* head receives a synthetic `queued` disposition instead of waiting.
* - Without one: forwarded normally; successful responses are normalized
* through the standard write path so dependent cached queries update.
@@ -107,6 +108,7 @@ const HYDRATION_DOCUMENT_CONTEXT_KEY = 'normalizedCacheHydrationDocument';
const QUEUE_REQUEST_TIMEOUT_MS = 60_000;
const QUEUE_LEASE_MS = 5 * 60_000;
const EMPTY_QUEUE_POLL_MS = 30_000;
+const MAX_MUTATION_SERVER_FAILURES = 10;
const NORMALIZED_CACHE_RESULT_METADATA_KEY = '__macroNormalizedCache';
@@ -169,6 +171,9 @@ type QueueAttemptContext = {
leaseOwner: string;
leaseGeneration: string;
attemptCount: number;
+ serverFailureCount: number;
+ /** Transport-only metadata; urql drops the response on error-free payloads. */
+ response?: Response;
};
function queueAttemptOf(op: Operation): QueueAttemptContext | undefined {
@@ -273,19 +278,22 @@ function retryDelayMs(attemptCount: number): number {
return Math.min(1_000 * 2 ** Math.max(0, attemptCount - 1), 60_000);
}
-/** Applies a hard network bound well inside the durable queue lease. */
+/** Bounds network time inside the lease and retains HTTP status for settlement. */
function withQueueRequestTimeout(op: Operation): Operation {
const operationFetch = op.context.fetch ?? globalThis.fetch;
+ const attempt = queueAttemptOf(op);
return makeOperation(op.kind, op, {
...op.context,
- fetch: (input, init) => {
+ fetch: async (input, init) => {
const timeoutSignal = AbortSignal.timeout(QUEUE_REQUEST_TIMEOUT_MS);
- return operationFetch(input, {
+ const response = await operationFetch(input, {
...init,
signal: init?.signal
? AbortSignal.any([init.signal, timeoutSignal])
: timeoutSignal,
});
+ if (attempt) attempt.response = response;
+ return response;
},
});
}
@@ -863,6 +871,7 @@ export function normalizedCacheExchange(
leaseOwner: queueOwner,
leaseGeneration: claimed.leaseGeneration,
attemptCount: claimed.attemptCount,
+ serverFailureCount: claimed.serverFailureCount ?? 0,
};
const live = liveQueuedOps.get(claimed.transactionId);
if (live) {
@@ -1473,6 +1482,20 @@ export function normalizedCacheExchange(
| 'superseded'
| 'permanently-failed' = 'queued';
try {
+ const response = result.error?.response ?? attempt.response;
+ const httpServerFailure = (response?.status ?? 0) >= 500;
+ // A valid GraphQL payload can arrive over HTTP 5xx without an
+ // urql error. Never commit it or bypass the retry policy.
+ if (httpServerFailure && !result.error) {
+ result = {
+ ...result,
+ data: undefined,
+ error: new CombinedError({
+ networkError: new Error(`HTTP ${response.status}`),
+ response,
+ }),
+ };
+ }
if (result.error || result.data == null) {
let retry = false;
if (result.error && options.shouldRetryMutation) {
@@ -1482,13 +1505,38 @@ export function normalizedCacheExchange(
options.onCacheError?.(error, op);
}
}
+ // urql represents HTTP 5xx as networkError too. Count actual
+ // server responses, never connection failures or timeouts.
+ const serverFailure =
+ httpServerFailure ||
+ (result.error?.graphQLErrors.length ?? 0) > 0;
+ if (
+ retry &&
+ serverFailure &&
+ attempt.serverFailureCount + 1 >= MAX_MUTATION_SERVER_FAILURES
+ ) {
+ retry = false;
+ result = {
+ ...result,
+ error: new CombinedError({
+ graphQLErrors: [
+ {
+ message: `Mutation stopped after ${MAX_MUTATION_SERVER_FAILURES} server failures: ${result.error?.message}`,
+ extensions: { code: 'MUTATION_RETRY_EXHAUSTED' },
+ },
+ ],
+ response: result.error?.response,
+ }),
+ };
+ }
if (retry) {
retryAt = Date.now() + retryDelayMs(attempt.attemptCount);
const deferred = await host.deferOptimisticWrite(
attempt.transactionId,
claim,
retryAt,
- result.error?.message ?? 'mutation returned no data'
+ result.error?.message ?? 'mutation returned no data',
+ serverFailure
);
if (deferred.kind === 'discarded-superseded') {
retryAt = undefined;
diff --git a/apps/web/src/lib/graphql-cache/host/retirable-host.test.ts b/apps/web/src/lib/graphql-cache/host/retirable-host.test.ts
index fc0c01d13b..48a132d59b 100644
--- a/apps/web/src/lib/graphql-cache/host/retirable-host.test.ts
+++ b/apps/web/src/lib/graphql-cache/host/retirable-host.test.ts
@@ -75,6 +75,28 @@ describe('createRetirableCacheHost', () => {
expect(inner.teardown).not.toHaveBeenCalled();
});
+ it('forwards retry accounting and permanent-failure codes', async () => {
+ const inner = createNoopCacheHost();
+ const defer = vi.spyOn(inner, 'deferOptimisticWrite');
+ const rollback = vi.spyOn(inner, 'rollbackOptimisticWrite');
+ const host = createRetirableCacheHost(inner);
+ const claim = { owner: 'runner', generation: '2' };
+ await host.deferOptimisticWrite('1', claim, 100, 'server failed', true);
+ expect(defer).toHaveBeenCalledWith('1', claim, 100, 'server failed', true);
+ await host.rollbackOptimisticWrite(
+ '1',
+ claim,
+ 'exhausted',
+ 'MUTATION_RETRY_EXHAUSTED'
+ );
+ expect(rollback).toHaveBeenCalledWith(
+ '1',
+ claim,
+ 'exhausted',
+ 'MUTATION_RETRY_EXHAUSTED'
+ );
+ });
+
it('retires once', () => {
const inner = disposedLikeHost();
const host = createRetirableCacheHost(inner);
diff --git a/apps/web/src/lib/graphql-cache/host/retirable-host.ts b/apps/web/src/lib/graphql-cache/host/retirable-host.ts
index f0c1946248..7b7ea957fb 100644
--- a/apps/web/src/lib/graphql-cache/host/retirable-host.ts
+++ b/apps/web/src/lib/graphql-cache/host/retirable-host.ts
@@ -30,17 +30,24 @@ export function createRetirableCacheHost(inner: CacheHost): CacheHost {
inspectQuery: (args) => current.inspectQuery(args),
claimNextMutation: (owner, nowMs, leaseExpiresAtMs) =>
current.claimNextMutation(owner, nowMs, leaseExpiresAtMs),
- deferOptimisticWrite: (transactionId, claim, nextAttemptAtMs, error) =>
+ deferOptimisticWrite: (
+ transactionId,
+ claim,
+ nextAttemptAtMs,
+ error,
+ serverFailure
+ ) =>
current.deferOptimisticWrite(
transactionId,
claim,
nextAttemptAtMs,
- error
+ error,
+ serverFailure
),
commitOptimisticWrite: (transactionId, claim, args) =>
current.commitOptimisticWrite(transactionId, claim, args),
- rollbackOptimisticWrite: (transactionId, claim, error) =>
- current.rollbackOptimisticWrite(transactionId, claim, error),
+ rollbackOptimisticWrite: (transactionId, claim, error, errorCode) =>
+ current.rollbackOptimisticWrite(transactionId, claim, error, errorCode),
invalidate: (keys) => current.invalidate(keys),
deleteRecords: (keys) => current.deleteRecords(keys),
teardown: (opKey) => current.teardown(opKey),
diff --git a/apps/web/src/lib/graphql-cache/host/tauri-host.ts b/apps/web/src/lib/graphql-cache/host/tauri-host.ts
index 3a398a911c..9941b529cf 100644
--- a/apps/web/src/lib/graphql-cache/host/tauri-host.ts
+++ b/apps/web/src/lib/graphql-cache/host/tauri-host.ts
@@ -433,7 +433,8 @@ export function createTauriCacheHost(options: TauriHostOptions): CacheHost {
transactionId: string,
claim: MutationClaim,
nextAttemptAtMs: number,
- error: string
+ error: string,
+ serverFailure = false
): Promise<DeferOptimisticWriteResult> {
await ready;
return await request<DeferOptimisticWriteResult>(
@@ -444,6 +445,7 @@ export function createTauriCacheHost(options: TauriHostOptions): CacheHost {
leaseGeneration: claim.generation,
nextAttemptAtMs,
error,
+ serverFailure,
}
);
},
diff --git a/apps/web/src/lib/graphql-cache/host/types.ts b/apps/web/src/lib/graphql-cache/host/types.ts
index 7cea57549c..08545b0799 100644
--- a/apps/web/src/lib/graphql-cache/host/types.ts
+++ b/apps/web/src/lib/graphql-cache/host/types.ts
@@ -153,7 +153,8 @@ export interface CacheHost {
transactionId: string,
claim: MutationClaim,
nextAttemptAtMs: number,
- error: string
+ error: string,
+ serverFailure?: boolean
): Promise<DeferOptimisticWriteResult>;
/** Atomically commits a claimed mutation's real network response. */
commitOptimisticWrite(
diff --git a/apps/web/src/lib/graphql-cache/host/worker-host.ts b/apps/web/src/lib/graphql-cache/host/worker-host.ts
index b02039b267..0d358ae2fa 100644
--- a/apps/web/src/lib/graphql-cache/host/worker-host.ts
+++ b/apps/web/src/lib/graphql-cache/host/worker-host.ts
@@ -1314,7 +1314,8 @@ export function createWorkerCacheHost(options: WorkerHostOptions): CacheHost {
transactionId: string,
claim: MutationClaim,
nextAttemptAtMs: number,
- error: string
+ error: string,
+ serverFailure = false
) {
return (await initializedRequest({
kind: 'defer-optimistic-write',
@@ -1323,6 +1324,7 @@ export function createWorkerCacheHost(options: WorkerHostOptions): CacheHost {
leaseGeneration: claim.generation,
nextAttemptAtMs,
error,
+ serverFailure,
})) as DeferOptimisticWriteResult;
},
diff --git a/apps/web/src/lib/graphql-cache/protocol.ts b/apps/web/src/lib/graphql-cache/protocol.ts
index 9bf468a99f..deb4eb9de2 100644
--- a/apps/web/src/lib/graphql-cache/protocol.ts
+++ b/apps/web/src/lib/graphql-cache/protocol.ts
@@ -417,6 +417,8 @@ export type ClaimedMutation = {
/** Identity witness captured at enqueue time. */
identity?: string;
attemptCount: number;
+ /** Retryable server failures; transport failures do not consume this budget. */
+ serverFailureCount: number;
};
/** Outcome of the strict-head claim attempted immediately after enqueue. */
@@ -554,6 +556,7 @@ export type CacheRequest = { id: number } & (
leaseGeneration: string;
nextAttemptAtMs: number;
error: string;
+ serverFailure?: boolean;
}
/** Atomically replace a claimed layer with the real network response. */
| {
diff --git a/apps/web/src/lib/graphql-cache/worker/browser-test/cache-lifecycle-harness.ts b/apps/web/src/lib/graphql-cache/worker/browser-test/cache-lifecycle-harness.ts
index de6dee4dba..c9a9724697 100644
--- a/apps/web/src/lib/graphql-cache/worker/browser-test/cache-lifecycle-harness.ts
+++ b/apps/web/src/lib/graphql-cache/worker/browser-test/cache-lifecycle-harness.ts
@@ -1,9 +1,16 @@
-import { type Client, createClient, gql, stringifyDocument } from '@urql/core';
+import {
+ type Client,
+ createClient,
+ fetchExchange,
+ gql,
+ stringifyDocument,
+} from '@urql/core';
import { filter, map, pipe } from 'wonka';
import { normalizedCacheExchange } from '../../exchange/normalized-cache-exchange';
import type { CacheHost } from '../../host/types';
import { createWorkerCacheHost } from '../../host/worker-host';
import { clearRegisteredCaches, registerCacheHost } from '../../lifecycle';
+import type { MutationSettlement } from '../../protocol';
const result = document.querySelector<HTMLElement>('#result');
if (!result) throw new Error('missing Cache result node');
@@ -227,6 +234,109 @@ const api = {
);
return queued.transactionId;
},
+ async seedRetryBudget(): Promise<string[]> {
+ const host = requireOwner();
+ const nowMs = Date.now();
+ const owner = 'retry-budget-seed';
+ const mutation = `mutation RetryBudget($input: SetEntityPropertyInput!) {
+ setEntityProperty(input: $input) { id displayName }
+ }`;
+ const enqueue = (id: string) =>
+ host.enqueueOptimisticMutation(
+ {
+ uuid: crypto.randomUUID(),
+ query: mutation,
+ variables: {
+ input: {
+ entityType: 'DOCUMENT',
+ entityId: id,
+ propertyDefinitionId: 'definition',
+ value: { string: id },
+ },
+ },
+ data: { setEntityProperty: { id, displayName: id } },
+ },
+ { owner, nowMs, leaseExpiresAtMs: nowMs + 300_000 }
+ );
+ const first = await enqueue('failing-property');
+ const second = await enqueue('following-property');
+ if (first.initialClaim.kind !== 'claimed')
+ throw new Error('missing first claim');
+ let claim = first.initialClaim.mutation;
+ for (let failure = 0; failure < 9; failure += 1) {
+ await host.deferOptimisticWrite(
+ first.transactionId,
+ { owner, generation: claim.leaseGeneration },
+ nowMs,
+ 'server failure',
+ true
+ );
+ if (failure < 8) {
+ const next = await host.claimNextMutation(
+ owner,
+ nowMs,
+ nowMs + 300_000
+ );
+ if (!next) throw new Error('missing retry claim');
+ claim = next;
+ }
+ }
+ // Transport failure after nine server failures must preserve the budget.
+ const network = await host.claimNextMutation(owner, nowMs, nowMs + 300_000);
+ if (!network || network.serverFailureCount !== 9)
+ throw new Error('budget not persisted');
+ await host.deferOptimisticWrite(
+ first.transactionId,
+ { owner, generation: network.leaseGeneration },
+ nowMs,
+ 'offline',
+ false
+ );
+ return [first.transactionId, second.transactionId];
+ },
+ async exhaustRetryBudget(): Promise<{
+ attempts: number;
+ settlements: MutationSettlement[];
+ }> {
+ const host = requireOwner();
+ const complete = Promise.withResolvers<void>();
+ const settlements: MutationSettlement[] = [];
+ let attempts = 0;
+ const unsubscribe = host.onMutationSettled((settlement) => {
+ settlements.push(settlement);
+ if (settlements.length === 2) complete.resolve();
+ });
+ mutationRunner = createClient({
+ url: 'http://cache-lifecycle.test/graphql',
+ fetch: async () => {
+ attempts += 1;
+ // Even a valid data payload over HTTP 5xx must exhaust the budget.
+ return new Response(
+ JSON.stringify({
+ data: {
+ setEntityProperty: {
+ id: attempts === 1 ? 'failing-property' : 'following-property',
+ displayName: 'Synced',
+ },
+ },
+ }),
+ {
+ status: attempts === 1 ? 503 : 200,
+ headers: { 'content-type': 'application/json' },
+ }
+ );
+ },
+ exchanges: [
+ normalizedCacheExchange(host, {
+ shouldRetryMutation: (error) => (error.response?.status ?? 0) >= 500,
+ }),
+ fetchExchange,
+ ],
+ });
+ await complete.promise;
+ unsubscribe();
+ return { attempts, settlements };
+ },
navigationState() {
return {
clientId: requireOwner().clientId,
diff --git a/apps/web/src/lib/graphql-cache/worker/browser-test/mutation-retry.browser.e2e.ts b/apps/web/src/lib/graphql-cache/worker/browser-test/mutation-retry.browser.e2e.ts
new file mode 100644
index 0000000000..d4c3f0b196
--- /dev/null
+++ b/apps/web/src/lib/graphql-cache/worker/browser-test/mutation-retry.browser.e2e.ts
@@ -0,0 +1,33 @@
+import { expect, test } from '@playwright/test';
+
+test('Cache exhausts a persisted server retry budget after reload and advances the queue', async ({
+ page,
+}, testInfo) => {
+ const path = testInfo.project.name.includes('production')
+ ? '/app/cache-lifecycle.html'
+ : '/cache-lifecycle.html';
+ const scope = `cache-retry-budget-${crypto.randomUUID()}`;
+ await page.goto(`${path}?treatment=true&scope=${scope}`);
+ await expect(page.locator('#result')).toHaveAttribute('data-status', 'ready');
+ await page.evaluate(() => window.cacheLifecycleHarness.startSingle());
+ const ids = await page.evaluate(() =>
+ window.cacheLifecycleHarness.seedRetryBudget()
+ );
+ await page.evaluate(() => window.cacheLifecycleHarness.dispose());
+ await page.reload();
+ await expect(page.locator('#result')).toHaveAttribute('data-status', 'ready');
+ await page.evaluate(() => window.cacheLifecycleHarness.startSingle());
+ const result = await page.evaluate(() =>
+ window.cacheLifecycleHarness.exhaustRetryBudget()
+ );
+ expect(result.attempts).toBe(2);
+ expect(result.settlements).toMatchObject([
+ {
+ transactionId: ids[0],
+ status: 'permanently-failed',
+ errorCode: 'MUTATION_RETRY_EXHAUSTED',
+ },
+ { transactionId: ids[1], status: 'committed' },
+ ]);
+ await page.evaluate(() => window.cacheLifecycleHarness.dispose());
+});
diff --git a/apps/web/src/lib/graphql-cache/worker/browser-test/playwright.production.config.ts b/apps/web/src/lib/graphql-cache/worker/browser-test/playwright.production.config.ts
index 66dc8f73af..6381d38d2d 100644
--- a/apps/web/src/lib/graphql-cache/worker/browser-test/playwright.production.config.ts
+++ b/apps/web/src/lib/graphql-cache/worker/browser-test/playwright.production.config.ts
@@ -12,6 +12,7 @@ export default defineConfig({
'cache-artifact-compatibility.browser.e2e.ts',
'cache-wasm-packaging.browser.e2e.ts',
'cache-lifecycle.browser.e2e.ts',
+ 'mutation-retry.browser.e2e.ts',
],
timeout: 90_000,
fullyParallel: false,
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts
index 6b827da0ec..f51e380480 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.test.ts
@@ -57,6 +57,24 @@ it.each([undefined, 'DRAFT_ALREADY_SENT', 'INTERNAL', 42, null])(
}
);
+it.each([undefined, true, false, 'true', 1, null])(
+ 'validates retry-budget classification %j',
+ (serverFailure) => {
+ expect(
+ isCacheRequest({
+ id: 1,
+ kind: 'defer-optimistic-write',
+ transactionId: '1',
+ leaseOwner: 'runner',
+ leaseGeneration: '1',
+ nextAttemptAtMs: 100,
+ error: 'failed',
+ serverFailure,
+ })
+ ).toBe(serverFailure === undefined || typeof serverFailure === 'boolean');
+ }
+);
+
const enginePort = {
postMessage() {},
close() {},
diff --git a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts
index efe4448929..6d98860bbe 100644
--- a/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts
+++ b/apps/web/src/lib/graphql-cache/worker/coordinator-protocol.ts
@@ -623,12 +623,15 @@ export function isCacheRequest(value: unknown): value is CacheRequest {
'leaseGeneration',
'nextAttemptAtMs',
'error',
+ 'serverFailure',
]) &&
isString(value.transactionId) &&
isString(value.leaseOwner) &&
isString(value.leaseGeneration) &&
isSafeNonNegativeInteger(value.nextAttemptAtMs) &&
- isString(value.error)
+ isString(value.error) &&
+ (value.serverFailure === undefined ||
+ typeof value.serverFailure === 'boolean')
);
case 'commit-optimistic-write':
return (
diff --git a/apps/web/src/lib/graphql-cache/worker/wasm-module.ts b/apps/web/src/lib/graphql-cache/worker/wasm-module.ts
index d27b53684b..476470bf80 100644
--- a/apps/web/src/lib/graphql-cache/worker/wasm-module.ts
+++ b/apps/web/src/lib/graphql-cache/worker/wasm-module.ts
@@ -152,7 +152,8 @@ export interface CacheEngine {
leaseOwner: string,
leaseGeneration: string,
nextAttemptAtMs: number,
- error: string
+ error: string,
+ serverFailure: boolean
): Promise<DeferOptimisticWriteResult>;
commitOptimisticWrite(
transactionId: string,
diff --git a/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts b/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts
index 33c0620907..c4eb34279e 100644
--- a/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts
+++ b/apps/web/src/lib/graphql-cache/worker/worker-core.test.ts
@@ -375,6 +375,37 @@ describe('CacheWorkerCore', () => {
});
});
+ it.each([undefined, false, true])(
+ 'forwards server-failure accounting to wasm: %s',
+ async (serverFailure) => {
+ const deferOptimisticWrite = vi.fn(async () => ({ kind: 'deferred' }));
+ loadCacheWasmMock.mockResolvedValue({
+ openCache: vi.fn().mockResolvedValue({ deferOptimisticWrite }),
+ });
+ const port = { postMessage: vi.fn() };
+ const core = new CacheWorkerCore();
+ await core.handleRequest(port, { id: 1, kind: 'init', scope: 'scope-1' });
+ await core.handleRequest(port, {
+ id: 2,
+ kind: 'defer-optimistic-write',
+ transactionId: '1',
+ leaseOwner: 'runner',
+ leaseGeneration: '2',
+ nextAttemptAtMs: 100,
+ error: 'failed',
+ serverFailure,
+ });
+ expect(deferOptimisticWrite).toHaveBeenCalledExactlyOnceWith(
+ '1',
+ 'runner',
+ '2',
+ 100,
+ 'failed',
+ serverFailure ?? false
+ );
+ }
+ );
+
it('runs standalone claims ahead of queued observational reads', async () => {
const order: string[] = [];
let releaseBlocker!: () => void;
diff --git a/apps/web/src/lib/graphql-cache/worker/worker-core.ts b/apps/web/src/lib/graphql-cache/worker/worker-core.ts
index 782aac03f0..e7e7aff074 100644
--- a/apps/web/src/lib/graphql-cache/worker/worker-core.ts
+++ b/apps/web/src/lib/graphql-cache/worker/worker-core.ts
@@ -613,7 +613,8 @@ export class CacheWorkerCore {
request.leaseOwner,
request.leaseGeneration,
request.nextAttemptAtMs,
- request.error
+ request.error,
+ request.serverFailure ?? false
);
if (result.kind === 'discarded-superseded') {
result.revision = parseCacheRevision(result.revision);
diff --git a/apps/web/tauri/graphql_cache_plugin/src/commands.rs b/apps/web/tauri/graphql_cache_plugin/src/commands.rs
index 44b96477bc..4af03abbf1 100644
--- a/apps/web/tauri/graphql_cache_plugin/src/commands.rs
+++ b/apps/web/tauri/graphql_cache_plugin/src/commands.rs
@@ -408,6 +408,7 @@ pub async fn graphql_cache_defer_optimistic_write<R: Runtime>(
lease_generation: String,
next_attempt_at_ms: i64,
error: String,
+ server_failure: Option<bool>,
) -> Result<DeferOptimisticWriteResultWire, String> {
let settlement_transaction_id = transaction_id.clone();
let result = engine_handle(&state)?
@@ -417,6 +418,7 @@ pub async fn graphql_cache_defer_optimistic_write<R: Runtime>(
lease_generation,
next_attempt_at_ms,
error,
+ server_failure.unwrap_or(false),
)
.await?;
if let DeferOptimisticWriteResultWire::DiscardedSuperseded {
diff --git a/apps/web/tauri/graphql_cache_plugin/src/engine.rs b/apps/web/tauri/graphql_cache_plugin/src/engine.rs
index 6599c0cc01..2b80e076cb 100644
--- a/apps/web/tauri/graphql_cache_plugin/src/engine.rs
+++ b/apps/web/tauri/graphql_cache_plugin/src/engine.rs
@@ -205,6 +205,8 @@ pub struct ClaimedMutationWire {
pub identity: Option<String>,
/// Number of network attempts including this claim.
pub attempt_count: u32,
+ /// Retryable server failures, excluding transport failures.
+ pub server_failure_count: u32,
}
/// Tagged result of deferring or discarding a failed queue attempt.
@@ -303,6 +305,7 @@ impl TryFrom<ClaimedMutation> for ClaimedMutationWire {
variables: serde_json::from_str(&request.variables_json).map_err(|e| e.to_string())?,
identity: request.identity,
attempt_count: claimed.queued.mutation.attempt_count,
+ server_failure_count: claimed.queued.mutation.server_failure_count,
})
}
}
@@ -770,6 +773,7 @@ impl EngineHandle {
lease_generation: String,
next_attempt_at_ms: i64,
error: String,
+ server_failure: bool,
) -> Result<DeferOptimisticWriteResultWire, String> {
let transaction = parse_transaction_id(&transaction_id)?;
let claim = MutationClaimToken {
@@ -779,7 +783,13 @@ impl EngineHandle {
let mut state = self.inner.lock().await;
let EngineState { engine, ops, .. } = &mut *state;
match engine
- .defer_optimistic_write(transaction, claim, next_attempt_at_ms, error)
+ .defer_optimistic_write(
+ transaction,
+ claim,
+ next_attempt_at_ms,
+ error,
+ server_failure,
+ )
.await
.map_err(|e| e.to_string())?
{
diff --git a/crates/client/README.md b/crates/client/README.md
index 676955ab3f..78b3ab479b 100644
--- a/crates/client/README.md
+++ b/crates/client/README.md
@@ -252,4 +252,10 @@ wipes + rebinds atomically when the tag changes (silent restart).
Optimistic GraphQL mutations are persisted with their replay request before
becoming visible. The exchange claims and applies them strictly in enqueue
order; a configurable callback decides whether an error remains queued or
-permanently rolls back.
+permanently rolls back. The exchange caps retryable server failures at ten per
+mutation, excluding transport failures/timeouts. This separate budget is stored
+in the existing Turso `meta` table under `mutation-server-failures:<id>` and
+updated atomically with fenced deferral; absent metadata starts at zero for
+existing queued writes. Commit, rollback, replacement, and clear remove it.
+The storage schema/database identity and the all-attempt backoff counter stay
+unchanged. Failed mutations are not retained in a DLQ.
diff --git a/crates/client/cache-core/src/engine.rs b/crates/client/cache-core/src/engine.rs
index 660e562fc9..ddd13d5979 100644
--- a/crates/client/cache-core/src/engine.rs
+++ b/crates/client/cache-core/src/engine.rs
@@ -1902,6 +1902,7 @@ impl<S: Storage> Engine<S> {
claim: MutationClaimToken,
next_attempt_at_ms: i64,
error: String,
+ server_failure: bool,
) -> Result<DeferOptimisticWriteResult, EngineError<S::Error>> {
self.hydrate_optimistic().await?;
let layer = self
@@ -1931,7 +1932,13 @@ impl<S: Storage> Engine<S> {
}
if !self
.storage
- .defer_mutation(transaction, claim, next_attempt_at_ms, error)
+ .defer_mutation(
+ transaction,
+ claim,
+ next_attempt_at_ms,
+ error,
+ server_failure,
+ )
.await
.map_err(EngineError::Storage)?
{
diff --git a/crates/client/cache-core/src/queue.rs b/crates/client/cache-core/src/queue.rs
index baf0eaae01..54e55ea348 100644
--- a/crates/client/cache-core/src/queue.rs
+++ b/crates/client/cache-core/src/queue.rs
@@ -37,6 +37,9 @@ pub struct StoredMutation {
pub request: MutationRequest,
/// Number of times this mutation has been claimed for a network attempt.
pub attempt_count: u32,
+ /// Retryable server failures accepted for this mutation, excluding transport failures.
+ #[serde(default)]
+ pub server_failure_count: u32,
/// Earliest wall-clock time at which the head may be claimed again.
pub next_attempt_at_ms: Option<i64>,
/// Owner of the current claim, if any.
@@ -57,6 +60,7 @@ impl StoredMutation {
Self {
request,
attempt_count: 0,
+ server_failure_count: 0,
next_attempt_at_ms: None,
lease_owner: None,
lease_generation: 0,
diff --git a/crates/client/cache-core/src/store.rs b/crates/client/cache-core/src/store.rs
index 8fbfa495c9..7d436c9d81 100644
--- a/crates/client/cache-core/src/store.rs
+++ b/crates/client/cache-core/src/store.rs
@@ -207,14 +207,15 @@ pub trait Storage: MaybeSend {
) -> impl Future<Output = Result<Option<ClaimedMutation>, Self::Error>> + MaybeSend;
/// Retains a retryable mutation and its optimistic layer, releases its
- /// lease, and records the next eligible attempt time. Returns `false`
- /// when the claim is stale.
+ /// lease, and records the next eligible attempt time. Server failures also
+ /// increment the retry budget atomically. Returns `false` when the claim is stale.
fn defer_mutation(
&mut self,
id: MutationId,
claim: MutationClaimToken,
next_attempt_at_ms: i64,
error: String,
+ server_failure: bool,
) -> impl Future<Output = Result<bool, Self::Error>> + MaybeSend;
/// Atomically writes the real response records and removes the mutation
@@ -635,6 +636,7 @@ impl Storage for InMemoryStorage {
claim: MutationClaimToken,
next_attempt_at_ms: i64,
error: String,
+ server_failure: bool,
) -> Result<bool, Self::Error> {
let Some(queued) = self.mutations.get_mut(&id) else {
return Ok(false);
@@ -645,6 +647,9 @@ impl Storage for InMemoryStorage {
let mutation = &mut queued.mutation;
mutation.next_attempt_at_ms = Some(next_attempt_at_ms);
mutation.last_error = Some(error);
+ if server_failure {
+ mutation.server_failure_count = mutation.server_failure_count.saturating_add(1);
+ }
mutation.lease_owner = None;
mutation.lease_expires_at_ms = None;
Ok(true)
diff --git a/crates/client/cache-core/tests/mutation_queue.rs b/crates/client/cache-core/tests/mutation_queue.rs
index 6dd07bcb50..401c55dab7 100644
--- a/crates/client/cache-core/tests/mutation_queue.rs
+++ b/crates/client/cache-core/tests/mutation_queue.rs
@@ -271,6 +271,7 @@ fn queue_claim_retry_and_settlement_are_ordered() {
},
200,
"offline".into(),
+ false,
)
.await
.unwrap()
diff --git a/crates/client/cache-core/tests/optimistic.rs b/crates/client/cache-core/tests/optimistic.rs
index 964559fd6a..3fe5f0e4f2 100644
--- a/crates/client/cache-core/tests/optimistic.rs
+++ b/crates/client/cache-core/tests/optimistic.rs
@@ -155,10 +155,11 @@ impl Storage for ClaimFailingStorage {
claim: MutationClaimToken,
next_attempt_at_ms: i64,
error: String,
+ server_failure: bool,
) -> Result<bool, Self::Error> {
Ok(self
.inner
- .defer_mutation(id, claim, next_attempt_at_ms, error)
+ .defer_mutation(id, claim, next_attempt_at_ms, error, server_failure)
.await
.unwrap())
}
@@ -560,7 +561,7 @@ fn enqueue_does_not_skip_a_leased_or_deferred_head() {
let (_, claim) = claim_head(&mut engine, "first-runner", 10).await;
if deferred {
engine
- .defer_optimistic_write(older, claim, 500, "offline".into())
+ .defer_optimistic_write(older, claim, 500, "offline".into(), false)
.await
.unwrap();
}
@@ -826,7 +827,7 @@ fn retryable_failure_keeps_optimistic_layer_and_blocks_later_mutations() {
let (_, claim) = claim_head(&mut engine, "runner", 10).await;
engine
- .defer_optimistic_write(first, claim, 100, "offline".into())
+ .defer_optimistic_write(first, claim, 100, "offline".into(), false)
.await
.unwrap();
assert!(
@@ -1085,6 +1086,7 @@ fn active_uuid_replacement_is_superseded_and_failed_attempt_is_discarded() {
},
500,
"superseded".into(),
+ false,
)
.await
.unwrap();
diff --git a/crates/client/cache-core/tests/query_inspection.rs b/crates/client/cache-core/tests/query_inspection.rs
index ec3368fd74..9b328cb456 100644
--- a/crates/client/cache-core/tests/query_inspection.rs
+++ b/crates/client/cache-core/tests/query_inspection.rs
@@ -147,9 +147,10 @@ impl Storage for OwnerOnlyStorage {
claim: MutationClaimToken,
next_attempt_at_ms: i64,
error: String,
+ server_failure: bool,
) -> Result<bool, Self::Error> {
self.0
- .defer_mutation(id, claim, next_attempt_at_ms, error)
+ .defer_mutation(id, claim, next_attempt_at_ms, error, server_failure)
.await
}
diff --git a/crates/client/cache-turso/src/storage.rs b/crates/client/cache-turso/src/storage.rs
index c426fb49c0..17d9af046a 100644
--- a/crates/client/cache-turso/src/storage.rs
+++ b/crates/client/cache-turso/src/storage.rs
@@ -133,8 +133,8 @@ const SORT_FACT_INSERT: &str =
"INSERT INTO sort_facts (document_id, attribute, value) VALUES (?1, ?2, ?3)";
const QUEUE_INSERT: &str = "INSERT INTO mutation_queue (uuid, superseded, query, operation_name, variables_json, identity, attempt_count, next_attempt_at_ms, lease_owner, lease_generation, lease_expires_at_ms, last_error, created_at_ms) VALUES (?1, 0, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12)";
const LAYER_INSERT: &str = "INSERT INTO optimistic_layers (mutation_id, optimistic_data_json, normalized_updates) VALUES (?1, ?2, ?3)";
-const QUEUE_SELECT: &str = "SELECT m.id, m.uuid, m.superseded, m.query, m.operation_name, m.variables_json, m.identity, m.attempt_count, m.next_attempt_at_ms, m.lease_owner, m.lease_generation, m.lease_expires_at_ms, m.last_error, m.created_at_ms, o.optimistic_data_json, o.normalized_updates FROM mutation_queue AS m LEFT JOIN optimistic_layers AS o ON o.mutation_id = m.id ORDER BY m.id ASC";
-const QUEUE_HEAD_SELECT: &str = "SELECT m.id, m.uuid, m.superseded, m.query, m.operation_name, m.variables_json, m.identity, m.attempt_count, m.next_attempt_at_ms, m.lease_owner, m.lease_generation, m.lease_expires_at_ms, m.last_error, m.created_at_ms, o.optimistic_data_json, o.normalized_updates FROM mutation_queue AS m LEFT JOIN optimistic_layers AS o ON o.mutation_id = m.id ORDER BY m.id ASC LIMIT 1";
+const QUEUE_SELECT: &str = "SELECT m.id, m.uuid, m.superseded, m.query, m.operation_name, m.variables_json, m.identity, m.attempt_count, m.next_attempt_at_ms, m.lease_owner, m.lease_generation, m.lease_expires_at_ms, m.last_error, m.created_at_ms, o.optimistic_data_json, o.normalized_updates, r.value FROM mutation_queue AS m LEFT JOIN optimistic_layers AS o ON o.mutation_id = m.id LEFT JOIN meta AS r ON r.key = 'mutation-server-failures:' || m.id ORDER BY m.id ASC";
+const QUEUE_HEAD_SELECT: &str = "SELECT m.id, m.uuid, m.superseded, m.query, m.operation_name, m.variables_json, m.identity, m.attempt_count, m.next_attempt_at_ms, m.lease_owner, m.lease_generation, m.lease_expires_at_ms, m.last_error, m.created_at_ms, o.optimistic_data_json, o.normalized_updates, r.value FROM mutation_queue AS m LEFT JOIN optimistic_layers AS o ON o.mutation_id = m.id LEFT JOIN meta AS r ON r.key = 'mutation-server-failures:' || m.id ORDER BY m.id ASC LIMIT 1";
const ORPHAN_LAYER_SELECT: &str = "SELECT o.mutation_id FROM optimistic_layers AS o LEFT JOIN mutation_queue AS m ON m.id = o.mutation_id WHERE m.id IS NULL LIMIT 1";
const ANY_LAYER_SELECT: &str = "SELECT mutation_id FROM optimistic_layers LIMIT 1";
const CLAIM_SELECT: &str = "SELECT lease_owner, lease_generation FROM mutation_queue WHERE id = ?1";
@@ -1041,6 +1041,7 @@ impl Storage for TursoStorage {
)?,
1,
)?;
+ mutation_retry::remove(&connection, mutation_id_to_sql(existing)?)?;
MutationUpsertKind::ReplacedPending {
removed_id: existing,
}
@@ -1054,6 +1055,11 @@ impl Storage for TursoStorage {
1,
)?;
let id = mutation_id_from_row(connection.last_insert_rowid())?;
+ mutation_retry::save(
+ &connection,
+ mutation_id_to_sql(id)?,
+ entry.mutation.server_failure_count,
+ )?;
self.fault_after(TestFaultSite::Enqueue, 1)?;
require_changed(
driver::execute(
@@ -1227,6 +1233,7 @@ impl Storage for TursoStorage {
claim: MutationClaimToken,
next_attempt_at_ms: i64,
error: String,
+ server_failure: bool,
) -> Result<bool, Self::Error> {
self.require_healthy()?;
let result = (|| {
@@ -1245,7 +1252,11 @@ impl Storage for TursoStorage {
text(&error),
],
)?;
- changed_to_bool(changed)
+ let accepted = changed_to_bool(changed)?;
+ if accepted && server_failure {
+ mutation_retry::increment(&connection, id)?;
+ }
+ Ok(accepted)
})
})();
self.latch_result(result)
@@ -1306,6 +1317,7 @@ impl Storage for TursoStorage {
)?,
1,
)?;
+ mutation_retry::remove(&connection, sql_id)?;
Ok(true)
})
})();
@@ -1361,6 +1373,7 @@ impl Storage for TursoStorage {
)?,
1,
)?;
+ mutation_retry::remove(&connection, sql_id)?;
write_shadow_reconciliation(&connection, reconciliation, |index| {
self.fault_after(TestFaultSite::Complete, entries.len() + index)
})?;
@@ -1393,6 +1406,7 @@ impl Storage for TursoStorage {
)?,
1,
)?;
+ mutation_retry::remove(&connection, sql_id)?;
self.fault_after(TestFaultSite::Discard, 0)?;
Ok(true)
})
@@ -1427,6 +1441,7 @@ impl Storage for TursoStorage {
)?,
1,
)?;
+ mutation_retry::remove(&connection, sql_id)?;
write_shadow_reconciliation(&connection, reconciliation, |index| {
self.fault_after(TestFaultSite::Discard, index)
})?;
@@ -1444,6 +1459,7 @@ impl Storage for TursoStorage {
driver::execute(&connection, "DELETE FROM optimistic_layers", Vec::new())?;
self.fault_after(TestFaultSite::Clear, 0)?;
driver::execute(&connection, "DELETE FROM mutation_queue", Vec::new())?;
+ mutation_retry::clear(&connection)?;
self.fault_after(TestFaultSite::Clear, 1)?;
driver::execute(&connection, "DELETE FROM search_documents", Vec::new())?;
driver::execute(&connection, "DELETE FROM index_documents", Vec::new())?;
@@ -4678,7 +4694,7 @@ struct ParsedQueueRow {
}
fn parse_queue_row(row: &[Value]) -> Result<ParsedQueueRow, TursoStorageError> {
- if row.len() != 16 {
+ if row.len() != 17 {
return Err(invariant());
}
let optimistic = match (&row[14], &row[15]) {
@@ -4702,6 +4718,7 @@ fn parse_queue_row(row: &[Value]) -> Result<ParsedQueueRow, TursoStorageError> {
identity: nullable_text(row, 6)?,
},
attempt_count: u32::try_from(required_i64(row, 7)?).map_err(|_| invariant())?,
+ server_failure_count: mutation_retry::parse(nullable_text(row, 16)?)?,
next_attempt_at_ms: nullable_i64(row, 8)?,
lease_owner: nullable_text(row, 9)?,
lease_generation: u64::try_from(required_i64(row, 10)?).map_err(|_| invariant())?,
@@ -4953,6 +4970,7 @@ mod alternatives;
mod bounded_selection;
mod conjunction;
mod integrity;
+mod mutation_retry;
mod page_retention;
mod record_batch;
diff --git a/crates/client/cache-turso/src/storage/mutation_retry.rs b/crates/client/cache-turso/src/storage/mutation_retry.rs
new file mode 100644
index 0000000000..2f07ccc333
--- /dev/null
+++ b/crates/client/cache-turso/src/storage/mutation_retry.rs
@@ -0,0 +1,63 @@
+//! Pending-mutation retry budgets in the existing metadata table. Keeping the
+//! frozen storage schema and database identity unchanged preserves queued writes
+//! across upgrades. All writes here run inside the queue's fenced transaction.
+
+use super::{Connection, TursoStorageError, driver, invariant, nullable_text, text};
+use std::sync::Arc;
+
+const KEY_PREFIX: &str = "mutation-server-failures:";
+
+pub(super) fn parse(value: Option<String>) -> Result<u32, TursoStorageError> {
+ value
+ .map(|value| value.parse().map_err(|_| invariant()))
+ .transpose()
+ .map(Option::unwrap_or_default)
+}
+
+pub(super) fn save(
+ connection: &Arc<Connection>,
+ id: i64,
+ count: u32,
+) -> Result<(), TursoStorageError> {
+ if count == 0 {
+ return Ok(());
+ }
+ driver::execute(
+ connection,
+ "INSERT INTO meta (key, value) VALUES (?1, ?2) ON CONFLICT (key) DO UPDATE SET value = excluded.value",
+ vec![text(&format!("{KEY_PREFIX}{id}")), text(&count.to_string())],
+ )?;
+ Ok(())
+}
+
+pub(super) fn increment(connection: &Arc<Connection>, id: i64) -> Result<(), TursoStorageError> {
+ let rows = driver::query(
+ connection,
+ "SELECT value FROM meta WHERE key = ?1",
+ vec![text(&format!("{KEY_PREFIX}{id}"))],
+ )?;
+ let count = match rows.as_slice() {
+ [] => 0,
+ [row] => parse(nullable_text(row, 0)?)?,
+ _ => return Err(invariant()),
+ };
+ save(connection, id, count.saturating_add(1))
+}
+
+pub(super) fn remove(connection: &Arc<Connection>, id: i64) -> Result<(), TursoStorageError> {
+ driver::execute(
+ connection,
+ "DELETE FROM meta WHERE key = ?1",
+ vec![text(&format!("{KEY_PREFIX}{id}"))],
+ )?;
+ Ok(())
+}
+
+pub(super) fn clear(connection: &Arc<Connection>) -> Result<(), TursoStorageError> {
+ driver::execute(
+ connection,
+ "DELETE FROM meta WHERE key GLOB ?1",
+ vec![text(&format!("{KEY_PREFIX}*"))],
+ )?;
+ Ok(())
+}
diff --git a/crates/client/cache-turso/src/storage/test.rs b/crates/client/cache-turso/src/storage/test.rs
index 1e289322f2..638b12898c 100644
--- a/crates/client/cache-turso/src/storage/test.rs
+++ b/crates/client/cache-turso/src/storage/test.rs
@@ -6,6 +6,7 @@ mod engine_writes;
mod fact_lookup_cost;
mod filter_scope_cost;
mod filter_scope_semantics;
+mod mutation_retry;
mod page_retention;
mod predicate_cost;
mod projection_writes;
@@ -168,7 +169,7 @@ async fn expect_every_storage_method_latched(
);
expect_reset_reason(
storage
- .defer_mutation(1, token("blocked", 1), 2, "blocked".into())
+ .defer_mutation(1, token("blocked", 1), 2, "blocked".into(), false)
.await,
expected,
);
diff --git a/crates/client/cache-turso/src/storage/test/mutation_retry.rs b/crates/client/cache-turso/src/storage/test/mutation_retry.rs
new file mode 100644
index 0000000000..08b57a2dc3
--- /dev/null
+++ b/crates/client/cache-turso/src/storage/test/mutation_retry.rs
@@ -0,0 +1,134 @@
+use super::*;
+
+fn budget_rows(storage: &TursoStorage) -> i64 {
+ raw_scalar(
+ storage,
+ "SELECT COUNT(*) FROM meta WHERE key GLOB 'mutation-server-failures:*'",
+ )
+}
+
+#[test]
+fn retry_budget_is_removed_with_its_pending_mutation() {
+ block_on(async {
+ for mode in ["commit", "rollback", "replace", "clear"] {
+ let database = TursoMemoryDatabase::new(format!("retry-cleanup-{mode}.db"));
+ let mut storage = database.open("retry-cleanup").unwrap();
+ let mut entry = queued("Pending");
+ let uuid = entry.uuid;
+ entry.mutation.server_failure_count = 9;
+ let id = storage.enqueue_mutation(entry).await.unwrap();
+ assert_eq!(budget_rows(&storage), 1);
+ match mode {
+ "replace" => {
+ let mut replacement = queued("Replacement");
+ replacement.uuid = uuid;
+ storage.enqueue_mutation(replacement).await.unwrap();
+ assert_eq!(
+ storage.load_mutation_queue().await.unwrap()[0]
+ .mutation
+ .server_failure_count,
+ 0
+ );
+ }
+ "clear" => storage.clear().await.unwrap(),
+ _ => {
+ let claim = storage
+ .claim_next_mutation(MutationClaimRequest {
+ owner: "runner".into(),
+ now_ms: 1,
+ lease_expires_at_ms: 2,
+ })
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(claim.queued.mutation.server_failure_count, 9);
+ let claim = token("runner", claim.lease_generation);
+ if mode == "commit" {
+ assert!(
+ storage
+ .complete_mutation(id, claim, Vec::new())
+ .await
+ .unwrap()
+ );
+ } else {
+ assert!(storage.discard_mutation(id, claim).await.unwrap());
+ }
+ }
+ }
+ assert_eq!(budget_rows(&storage), 0, "{mode}");
+ storage.try_close().unwrap();
+ }
+ });
+}
+
+#[test]
+fn retry_budget_survives_file_reopen_without_a_schema_change() {
+ block_on(async {
+ let directory = tempfile::tempdir().unwrap();
+ let path = directory.path().join("retry.db");
+ let database = TursoFileDatabase::new(&path).unwrap();
+ let mut storage = database.open("retry").unwrap();
+ let id = storage
+ .enqueue_mutation(queued("LegacyPending"))
+ .await
+ .unwrap();
+ assert_eq!(budget_rows(&storage), 0);
+ let claim = storage
+ .claim_next_mutation(MutationClaimRequest {
+ owner: "old-tab".into(),
+ now_ms: 1,
+ lease_expires_at_ms: 2,
+ })
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(claim.queued.mutation.server_failure_count, 0);
+ assert!(
+ storage
+ .defer_mutation(
+ id,
+ token("old-tab", claim.lease_generation),
+ 2,
+ "server failed".into(),
+ true
+ )
+ .await
+ .unwrap()
+ );
+ storage.try_close().unwrap();
+ drop(database);
+ let database = TursoFileDatabase::new(path).unwrap();
+ let mut storage = database.open("retry").unwrap();
+ let claim = storage
+ .claim_next_mutation(MutationClaimRequest {
+ owner: "new-tab".into(),
+ now_ms: 2,
+ lease_expires_at_ms: 3,
+ })
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(claim.queued.id, id);
+ assert_eq!(claim.queued.mutation.attempt_count, 2);
+ assert_eq!(claim.queued.mutation.server_failure_count, 1);
+ assert!(
+ storage
+ .defer_mutation(
+ id,
+ token("new-tab", claim.lease_generation),
+ 3,
+ "offline".into(),
+ false
+ )
+ .await
+ .unwrap()
+ );
+ assert_eq!(
+ storage.load_mutation_queue().await.unwrap()[0]
+ .mutation
+ .server_failure_count,
+ 1
+ );
+ storage.try_close().unwrap();
+ });
+}
diff --git a/crates/client/cache-turso/tests/draft_identity.rs b/crates/client/cache-turso/tests/draft_identity.rs
index 3751e12c0c..079a727a9a 100644
--- a/crates/client/cache-turso/tests/draft_identity.rs
+++ b/crates/client/cache-turso/tests/draft_identity.rs
@@ -168,7 +168,7 @@ fn disk_restart_rebases_newer_edits_and_preserves_uncertain_save_before_discard(
let first_claim = claim(&mut engine, first, 1).await;
assert!(matches!(
engine
- .defer_optimistic_write(first, first_claim, 20, "uncertain response".into())
+ .defer_optimistic_write(first, first_claim, 20, "uncertain response".into(), false)
.await
.unwrap(),
DeferOptimisticWriteResult::Deferred
@@ -219,7 +219,7 @@ fn disk_restart_rebases_newer_edits_and_preserves_uncertain_save_before_discard(
assert!(read(&mut engine, "local-draft").await.is_empty());
assert!(matches!(
engine
- .defer_optimistic_write(newer, newer_claim, 40, "uncertain edit".into())
+ .defer_optimistic_write(newer, newer_claim, 40, "uncertain edit".into(), false)
.await
.unwrap(),
DeferOptimisticWriteResult::Deferred
diff --git a/crates/client/cache-turso/tests/engine_turso.rs b/crates/client/cache-turso/tests/engine_turso.rs
index 8ca065f2b0..2e6c0ed19f 100644
--- a/crates/client/cache-turso/tests/engine_turso.rs
+++ b/crates/client/cache-turso/tests/engine_turso.rs
@@ -242,6 +242,7 @@ fn optimistic_hydration_retry_complete_and_reopen_run_over_turso() {
},
200,
"offline".into(),
+ false,
)
.await
.unwrap();
diff --git a/crates/client/cache-turso/tests/shared_conformance.rs b/crates/client/cache-turso/tests/shared_conformance.rs
index cbef13471e..ca87868800 100644
--- a/crates/client/cache-turso/tests/shared_conformance.rs
+++ b/crates/client/cache-turso/tests/shared_conformance.rs
@@ -106,6 +106,7 @@ fn fully_populated_queued() -> NewQueuedMutation {
identity: Some("identity-witness".into()),
},
attempt_count: 7,
+ server_failure_count: 3,
next_attempt_at_ms: Some(-2),
lease_owner: Some("expired-owner".into()),
lease_generation: 11,
@@ -343,7 +344,7 @@ async fn queue_contract<S: Storage>(storage: &mut S) {
let absent = token("absent", 1);
assert!(
!storage
- .defer_mutation(999, absent.clone(), 2, "absent".into())
+ .defer_mutation(999, absent.clone(), 2, "absent".into(), true)
.await
.unwrap()
);
@@ -397,6 +398,7 @@ async fn queue_contract<S: Storage>(storage: &mut S) {
token("runner-a", first_claim.lease_generation),
20,
"stale".into(),
+ true,
)
.await
.unwrap()
@@ -414,6 +416,7 @@ async fn queue_contract<S: Storage>(storage: &mut S) {
token("runner-b", expired_reclaim.lease_generation),
20,
"retry".into(),
+ true,
)
.await
.unwrap()
@@ -471,6 +474,7 @@ async fn queue_contract<S: Storage>(storage: &mut S) {
token("runner-d", second_claim.lease_generation),
30,
"second retry".into(),
+ false,
)
.await
.unwrap()
@@ -527,6 +531,58 @@ async fn queue_contract<S: Storage>(storage: &mut S) {
);
}
+async fn retry_budget_contract<F: BackendFactory>(
+ factory: &mut F,
+ mut storage: F::Backend,
+) -> F::Backend {
+ let id = storage
+ .enqueue_mutation(queued("RetryBudget"))
+ .await
+ .unwrap();
+ for (index, (server_failure, expected_count)) in
+ [(true, 1), (false, 1), (true, 2)].into_iter().enumerate()
+ {
+ let now = index as i64;
+ let claimed = storage
+ .claim_next_mutation(claim_request("runner", now, now + 1))
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(claimed.queued.id, id);
+ assert_eq!(claimed.queued.mutation.attempt_count, index as u32 + 1);
+ let claim = token("runner", claimed.lease_generation);
+ assert!(
+ storage
+ .defer_mutation(id, claim.clone(), now + 1, "retry".into(), server_failure)
+ .await
+ .unwrap()
+ );
+ // A duplicate result must not charge the same attempt twice.
+ assert!(
+ !storage
+ .defer_mutation(id, claim, now + 1, "duplicate".into(), true)
+ .await
+ .unwrap()
+ );
+ storage = factory.reopen(storage);
+ let queue = storage.load_mutation_queue().await.unwrap();
+ assert_eq!(queue[0].mutation.server_failure_count, expected_count);
+ }
+ let claimed = storage
+ .claim_next_mutation(claim_request("next-tab", 3, 4))
+ .await
+ .unwrap()
+ .unwrap();
+ assert_eq!(claimed.queued.mutation.server_failure_count, 2);
+ assert!(
+ storage
+ .discard_mutation(id, token("next-tab", claimed.lease_generation))
+ .await
+ .unwrap()
+ );
+ storage
+}
+
async fn clear_contract<S: Storage>(storage: &mut S) {
let before_clear = storage.enqueue_mutation(queued("Clear")).await.unwrap();
storage.clear().await.unwrap();
@@ -570,6 +626,7 @@ async fn common_contract<F: BackendFactory>(
search_projection_contract(&mut storage).await;
let mut storage = reopen_contract(factory, storage).await;
queue_contract(&mut storage).await;
+ let mut storage = retry_budget_contract(factory, storage).await;
clear_contract(&mut storage).await;
storage
}
diff --git a/crates/client/cache-turso/tests/storage_conformance.rs b/crates/client/cache-turso/tests/storage_conformance.rs
index 593e35a9d6..2b99a8d5fa 100644
--- a/crates/client/cache-turso/tests/storage_conformance.rs
+++ b/crates/client/cache-turso/tests/storage_conformance.rs
@@ -37,6 +37,7 @@ fn queued(label: &str, created_at_ms: i64) -> NewQueuedMutation {
identity: Some("identity-witness".into()),
},
attempt_count: 7,
+ server_failure_count: 3,
next_attempt_at_ms: None,
lease_owner: None,
lease_generation: 11,
@@ -307,13 +308,13 @@ fn strict_head_retries_leases_and_fences_conform() {
assert_eq!(reclaimed.lease_generation, 2);
assert!(
!storage
- .defer_mutation(first, claim("runner-a", 1), 300, "stale".into())
+ .defer_mutation(first, claim("runner-a", 1), 300, "stale".into(), true)
.await
.unwrap()
);
assert!(
storage
- .defer_mutation(first, claim("runner-b", 2), 300, "offline".into())
+ .defer_mutation(first, claim("runner-b", 2), 300, "offline".into(), false)
.await
.unwrap()
);
diff --git a/crates/client/cache-wasm/src/shell.rs b/crates/client/cache-wasm/src/shell.rs
index 45d9343497..07e471bad1 100644
--- a/crates/client/cache-wasm/src/shell.rs
+++ b/crates/client/cache-wasm/src/shell.rs
@@ -246,6 +246,7 @@ struct JsClaimedMutation {
variables: serde_json::Value,
identity: Option<String>,
attempt_count: u32,
+ server_failure_count: u32,
}
impl TryFrom<ClaimedMutation> for JsClaimedMutation {
@@ -265,6 +266,7 @@ impl TryFrom<ClaimedMutation> for JsClaimedMutation {
variables: serde_json::from_str(&request.variables_json).map_err(err_js)?,
identity: request.identity,
attempt_count: claimed.queued.mutation.attempt_count,
+ server_failure_count: claimed.queued.mutation.server_failure_count,
})
}
}
@@ -1863,6 +1865,7 @@ impl CacheEngine {
lease_generation: String,
next_attempt_at_ms: f64,
error: String,
+ server_failure: bool,
) -> js_sys::Promise {
let state = self.state.clone();
let ops = self.ops.clone();
@@ -1877,7 +1880,13 @@ impl CacheEngine {
let next_attempt_at_ms = parse_timestamp(next_attempt_at_ms, "next attempt timestamp")?;
let result = state
.engine_mut()?
- .defer_optimistic_write(transaction, claim, next_attempt_at_ms, error)
+ .defer_optimistic_write(
+ transaction,
+ claim,
+ next_attempt_at_ms,
+ error,
+ server_failure,
+ )
.await;
let result = state.engine_result(result)?;
let result = match result {
diff --git a/crates/client/cache-wasm/src/shell/test.rs b/crates/client/cache-wasm/src/shell/test.rs
index 091e50df90..233134b25a 100644
--- a/crates/client/cache-wasm/src/shell/test.rs
+++ b/crates/client/cache-wasm/src/shell/test.rs
@@ -2210,6 +2210,7 @@ async fn every_method_rejects_after_consuming_close() {
"1".into(),
2.0,
"closed".into(),
+ false,
))
.await;
assert_closed(engine.commit_optimistic_write(
diff --git a/crates/client/cache-wasm/src/shell/test_storage.rs b/crates/client/cache-wasm/src/shell/test_storage.rs
index 63ac7a119c..4f1c3cea14 100644
--- a/crates/client/cache-wasm/src/shell/test_storage.rs
+++ b/crates/client/cache-wasm/src/shell/test_storage.rs
@@ -156,9 +156,10 @@ impl Storage for BrowserStorage {
claim: MutationClaimToken,
next_attempt_at_ms: i64,
error: String,
+ server_failure: bool,
) -> Result<bool, Self::Error> {
self.inner
- .defer_mutation(id, claim, next_attempt_at_ms, error)
+ .defer_mutation(id, claim, next_attempt_at_ms, error, server_failure)
.await
}