Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Retry fast aggregate write conflicts within a bounded window (#1331)

Dee1204f92c3b8c
diff --git a/backend/src/lib/productUsage.test.ts b/backend/src/lib/productUsage.test.ts
index 3c16def8b..ad4b63a7e 100644
--- a/backend/src/lib/productUsage.test.ts
+++ b/backend/src/lib/productUsage.test.ts
@@ -1,5 +1,8 @@
import { beforeEach, describe, expect, it, vi } from 'vitest'
import { readFileSync } from 'node:fs'
+import { setTimeout as delay } from 'node:timers/promises'
+
+vi.mock('node:timers/promises', () => ({ setTimeout: vi.fn(async () => {}) }))
const db = vi.hoisted(() => ({
productUsagePreference: { findUnique: vi.fn(), upsert: vi.fn() },
@@ -99,6 +102,51 @@ describe('aggregate ingestion', () => {
db.productUsageTotal.upsert.mockRejectedValue({ code: 'P2002' })
db.productUsageReceipt.findUnique.mockResolvedValue(null)
await expect(ingestProductUsage('a', report, now)).rejects.toMatchObject({ code: 'P2002' })
+ expect(db.$transaction).toHaveBeenCalledTimes(4)
+ expect(delay).toHaveBeenCalledTimes(3)
+ })
+ it.each(['P2034', 'P2002'])('retries a fast %s conflict with the same receipt and a fresh permission check', async code => {
+ db.productUsageTotal.upsert.mockRejectedValueOnce({ code })
+ expect(await ingestProductUsage('private-account', report, now)).toEqual({ accepted: true })
+ expect(db.$transaction).toHaveBeenCalledTimes(2)
+ expect(db.productUsagePreference.findUnique).toHaveBeenCalledTimes(2)
+ expect(db.productUsageReceipt.create.mock.calls[0]).toEqual(db.productUsageReceipt.create.mock.calls[1])
+ expect(delay).toHaveBeenCalledExactlyOnceWith(expect.any(Number))
+ expect(vi.mocked(delay).mock.calls[0][0]).toBeGreaterThanOrEqual(25)
+ expect(vi.mocked(delay).mock.calls[0][0]).toBeLessThan(75)
+ })
+ it('rechecks a sharing-off choice before retrying the transaction', async () => {
+ db.productUsageTotal.upsert.mockRejectedValueOnce({ code: 'P2034' })
+ db.productUsagePreference.findUnique.mockResolvedValueOnce({ enabled: true, revision: 0 })
+ .mockResolvedValueOnce({ enabled: false, revision: 1 })
+ expect(await ingestProductUsage('a', report, now)).toMatchObject({ accepted: false, reason: 'CHOICE_CHANGED' })
+ expect(db.$transaction).toHaveBeenCalledTimes(2)
+ expect(db.productUsageReceipt.create).toHaveBeenCalledOnce()
+ })
+ it('bounds persistent conflicts and does not retry unrelated storage errors', async () => {
+ db.$transaction.mockRejectedValue({ code: 'P2034' })
+ await expect(ingestProductUsage('a', report, now)).rejects.toMatchObject({ code: 'P2034' })
+ expect(db.$transaction).toHaveBeenCalledTimes(4)
+ expect(delay).toHaveBeenCalledTimes(3)
+ vi.mocked(delay).mockClear(); db.$transaction.mockClear().mockRejectedValue({ code: 'P2028' })
+ await expect(ingestProductUsage('a', report, now)).rejects.toMatchObject({ code: 'P2028' })
+ expect(db.$transaction).toHaveBeenCalledOnce(); expect(delay).not.toHaveBeenCalled()
+ })
+ it('does not extend an already slow conflict with another transaction', async () => {
+ const clock = vi.spyOn(performance, 'now').mockReturnValueOnce(0).mockReturnValue(1000)
+ try {
+ db.$transaction.mockRejectedValue({ code: 'P2034' })
+ await expect(ingestProductUsage('a', report, now)).rejects.toMatchObject({ code: 'P2034' })
+ expect(db.$transaction).toHaveBeenCalledOnce(); expect(delay).not.toHaveBeenCalled()
+ } finally { clock.mockRestore() }
+ })
+ it('checks the retry window again after a delayed timer wakes', async () => {
+ const clock = vi.spyOn(performance, 'now').mockReturnValueOnce(0).mockReturnValueOnce(0).mockReturnValue(1000)
+ try {
+ db.$transaction.mockRejectedValue({ code: 'P2034' })
+ await expect(ingestProductUsage('a', report, now)).rejects.toMatchObject({ code: 'P2034' })
+ expect(db.$transaction).toHaveBeenCalledOnce(); expect(delay).toHaveBeenCalledOnce()
+ } finally { clock.mockRestore() }
})
it('fails closed when the preference cannot be checked', async () => {
db.productUsagePreference.findUnique.mockRejectedValue(new Error('unavailable'))
diff --git a/backend/src/lib/productUsage.ts b/backend/src/lib/productUsage.ts
index 0ce220636..9363868fd 100644
--- a/backend/src/lib/productUsage.ts
+++ b/backend/src/lib/productUsage.ts
@@ -1,6 +1,7 @@
import { healthBreakdowns } from '../analytics/productHealth.js'
import type { ProductUsageContext } from '../types/productHealthContract.js'
import { createHmac } from 'node:crypto'
+import { setTimeout as delay } from 'node:timers/promises'
import type { Prisma } from '@prisma/client'
import { env } from '../config/env.js'
import { prisma } from './prisma.js'
@@ -51,38 +52,49 @@ export async function ingestSetupUsage(report: SetupUsageReport, now = new Date(
async function ingestCounters(report: SetupUsageReport & { revision?: number; context?: ProductUsageContext }, userId?: string, now = new Date()) {
if (report.context?.channel === 'development') return { accepted: true, reason: 'DEVELOPMENT' }
- try {
- return await prisma.$transaction(async tx => {
- if (userId) {
- const choice = await productUsageChoice(userId, tx)
- if (!choice.enabled || choice.revision !== report.revision) return { accepted: false, reason: 'CHOICE_CHANGED', ...choice }
- }
- // Receipt and increments commit together; a lost HTTP response can be retried with the same id.
- await tx.productUsageReceipt.create({ data: {
- id: report.id, expiresAt: new Date(Date.parse(report.day) + (PRODUCT_USAGE_DAYS + 1) * DAY),
- } })
- for (const [metric, value] of Object.entries(report.counters)) {
- if (!value) continue
- for (const name of [metric, ...healthBreakdowns(report.context, metric)]) {
- await tx.productUsageTotal.upsert({
- where: { dayUtc_metric: { dayUtc: new Date(report.day), metric: name } },
- create: { dayUtc: new Date(report.day), metric: name, value }, update: { value: { increment: value } },
- })
+ // October's delivery audit found aggregate write conflicts. Retry the entire transaction,
+ // including permission, without extending a slow request or waiting a full client interval.
+ // At most four attempts; only fast conflicts may start another within the first second.
+ const started = performance.now()
+ for (let attempt = 0; ; attempt++) {
+ try {
+ return await prisma.$transaction(async tx => {
+ if (userId) {
+ const choice = await productUsageChoice(userId, tx)
+ if (!choice.enabled || choice.revision !== report.revision) return { accepted: false, reason: 'CHOICE_CHANGED', ...choice }
+ }
+ // Receipt and increments commit together; a lost HTTP response can be retried with the same id.
+ await tx.productUsageReceipt.create({ data: {
+ id: report.id, expiresAt: new Date(Date.parse(report.day) + (PRODUCT_USAGE_DAYS + 1) * DAY),
+ } })
+ for (const [metric, value] of Object.entries(report.counters)) {
+ if (!value) continue
+ for (const name of [metric, ...healthBreakdowns(report.context, metric)]) {
+ await tx.productUsageTotal.upsert({
+ where: { dayUtc_metric: { dayUtc: new Date(report.day), metric: name } },
+ create: { dayUtc: new Date(report.day), metric: name, value }, update: { value: { increment: value } },
+ })
+ }
}
- }
- if (report.context && report.day < now.toISOString().slice(0, 10)) {
- const dayUtc = new Date(Math.floor(now.getTime() / DAY) * DAY)
- const metric = `health_${report.context.surface}_report_late`
- await tx.productUsageTotal.upsert({ where: { dayUtc_metric: { dayUtc, metric } },
- create: { dayUtc, metric, value: 1 }, update: { value: { increment: 1 } } })
- }
- return { accepted: true }
- })
- } catch (error) {
- // A competing first writer of a daily total is NOT a duplicate report: let the client retry it.
- if ((error as { code?: string }).code === 'P2002'
- && await prisma.productUsageReceipt.findUnique({ where: { id: report.id }, select: { id: true } })) return { accepted: true }
- throw error
+ if (report.context && report.day < now.toISOString().slice(0, 10)) {
+ const dayUtc = new Date(Math.floor(now.getTime() / DAY) * DAY)
+ const metric = `health_${report.context.surface}_report_late`
+ await tx.productUsageTotal.upsert({ where: { dayUtc_metric: { dayUtc, metric } },
+ create: { dayUtc, metric, value: 1 }, update: { value: { increment: 1 } } })
+ }
+ return { accepted: true }
+ })
+ } catch (error) {
+ // A competing first writer of a daily total is not a duplicate unless its receipt committed.
+ const code = (error as { code?: string } | null)?.code
+ if (code === 'P2002'
+ && await prisma.productUsageReceipt.findUnique({ where: { id: report.id }, select: { id: true } })) return { accepted: true }
+ const pause = 25 * 2 ** attempt + Math.floor(Math.random() * 50)
+ if ((code !== 'P2034' && code !== 'P2002') || attempt >= 3
+ || performance.now() - started + pause >= 1000) throw error
+ await delay(pause)
+ if (performance.now() - started >= 1000) throw error
+ }
}
}
diff --git a/docs/product-measurement.md b/docs/product-measurement.md
index 7f4733a02..6233a26ac 100644
--- a/docs/product-measurement.md
+++ b/docs/product-measurement.md
@@ -115,6 +115,10 @@ Exhaustion drops measurement rather than affecting sign-in. Public counts remain
fabrication; they are directional diagnostics, not trusted account acquisition metrics.
Both ingestion paths atomically add only UTC-day/metric totals plus an unlinked random retry receipt.
+Fast database write conflicts retry that complete transaction up to three times with short jitter,
+rechecking the sharing choice each time. No retry starts after the first second; existing transaction
+deadlines still apply to each attempt. Slow or exhausted conflicts return to the client's normal
+immutable-report retry. Receipts prevent both immediate and later retries from counting twice.
Receipts expire within 36 days and totals within 396 days through the existing Mongo TTL indexes.
No account deduplication markers or stable hashes enter these totals. People-based daily counts and
D1/D2/D3/D7 returns come from one mark per sharing person per day under an HMAC pseudonym (no time,