Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

Backend: the usage-receipt sweep removes old receipts without its rule when the database refuses it, one chunk every 30 s (#2912)

0xken158af48019036b
diff --git a/backend/integration/productUsageReceipts.test.ts b/backend/integration/productUsageReceipts.test.ts
index 55112e8e6..aeecf3019 100644
--- a/backend/integration/productUsageReceipts.test.ts
+++ b/backend/integration/productUsageReceipts.test.ts
@@ -200,7 +200,7 @@ it('the worker\'s sweep runs a pass a minute and stops by itself once nothing is
expect(await left()).toEqual(KEPT.filter(id => !id.startsWith('old-2026-10-30')).sort())
})
-it('a failed pass is logged and the next minute\'s pass carries on', async () => {
+it('a failed pass is logged and the next pass carries on', async () => {
const lines: string[] = []
const log = { info: (message: string) => lines.push(message), warn: (message: string) => lines.push(message) }
await db.productUsageReceipt.deleteMany({ where: { id: { startsWith: 'old-2026-10-30' } } })
@@ -213,7 +213,7 @@ it('a failed pass is logged and the next minute\'s pass carries on', async () =>
try {
await vi.waitFor(() => expect(lines.at(-1)).toMatch(/done/), { timeout: 5000 })
} finally { stop() }
- expect(lines[0]).toMatch(/pass failed; retrying next minute/)
+ expect(lines[0]).toMatch(/pass failed; retrying next pass/)
expect(await left()).toEqual(KEPT.filter(id => !id.startsWith('old-2026-10-30')).sort())
})
@@ -233,37 +233,27 @@ function refusingCollMod(refused: () => boolean) {
/** Each line the sweep logs, with its level. */
const levels = (lines: string[]) => ({ info: (message: string) => lines.push(`info ${message}`), warn: (message: string) => lines.push(`warn ${message}`),
error: (message: string) => lines.push(`error ${message}`) })
-const STUCK = 'error [usage-receipts] stuck: the database user may not install the receipt rule (collMod), so nothing is removed. The old receipts ' +
- 'expire by themselves by 2026-12-04; grant collMod to finish sooner. Checking hourly.'
+const WITHOUT_RULE = 'warn [usage-receipts] the database user may not install the receipt rule (collMod): removing the old receipts ' +
+ 'without it (Ken, 2026-11-04). A server from before 2026-10-29, were one rolled back, could count a retried report twice.'
-it('a database user that may not install the rule gets an error every hourly check, then the sweep runs once it may (prod 10-29, task #2488)', async () => {
+it('a database user that may not install the rule asks once, says so once, and removes the old receipts anyway (Ken, 2026-11-04)', async () => {
const lines: string[] = []
await db.productUsageReceipt.deleteMany({ where: { id: { startsWith: 'old-2026-10-30' } } })
- let refused = true, clock = now.getTime()
- const { client, tries } = refusingCollMod(() => refused)
- const stop = sweep.startReceiptSweep(client, { everyMs: 20, chunk: 50, chunks: 1, pauseMs: 0, now: () => new Date(clock), log: levels(lines) })
+ const { client, tries } = refusingCollMod(() => true)
+ const stop = sweep.startReceiptSweep(client, { everyMs: 20, chunk: 50, chunks: 1, pauseMs: 0, now: () => now, log: levels(lines) })
try {
- await vi.waitFor(() => expect(tries.installs).toBe(1), { timeout: 5000 })
- // Many minutes' passes later, within the hour: no second try, no second line.
- await new Promise(resolve => setTimeout(resolve, 200))
- expect(tries.installs).toBe(1)
- expect(lines).toEqual([STUCK])
- expect((await left()).length).toBe(109)
- clock += 3_600_000
- await vi.waitFor(() => expect(tries.installs).toBe(2), { timeout: 5000 })
- await new Promise(resolve => setTimeout(resolve, 200))
- expect(tries.installs).toBe(2)
- // Every hour it is stuck says so again: one quiet warning at start was missed for days (#2245).
- expect(lines).toEqual([STUCK, STUCK])
- // Ken grants collMod: the next hourly check installs the rule and the sweep carries on, without a restart.
- refused = false
- clock += 3_600_000
await vi.waitFor(() => expect(lines.at(-1)).toMatch(/done/), { timeout: 5000 })
} finally { stop() }
+ expect(tries.installs).toBe(1)
+ expect(lines[0]).toBe(WITHOUT_RULE)
+ expect(lines.filter(line => line === WITHOUT_RULE)).toHaveLength(1)
expect(await left()).toEqual(KEPT.filter(id => !id.startsWith('old-2026-10-30')).sort())
+ const [info] = (await db.$runCommandRaw({ listCollections: 1, filter: { name: 'product_usage_receipts' } }) as
+ { cursor: { firstBatch: Array<{ options: Record<string, unknown> }> } }).cursor.firstBatch
+ expect(info.options).toMatchObject({ validationLevel: 'off' })
})
-it('a refused sweep stops by itself once its old receipts have expired, and asks for collMod no more (task #2488)', async () => {
+it('a refused sweep stops by itself once its old receipts have expired, and asks for collMod no more', async () => {
const lines: string[] = []
// Mongo's expiry has removed every old receipt the sweep would have: those it never touches (within 5 days) are left.
await db.productUsageReceipt.deleteMany({ where: { id: { startsWith: 'old-' }, expiresAt: { gt: at('2026-11-02', 5) } } })
@@ -275,8 +265,8 @@ it('a refused sweep stops by itself once its old receipts have expired, and asks
clock += 3 * 3_600_000
await new Promise(resolve => setTimeout(resolve, 200))
} finally { stop() }
- expect(lines).toEqual(['info [usage-receipts] done: no old receipts left'])
- expect(tries.installs).toBeLessThanOrEqual(1)
+ expect(lines).toEqual([WITHOUT_RULE, 'info [usage-receipts] done: no old receipts left'])
+ expect(tries.installs).toBe(1)
})
it('counting stays exact: a report inside the window counts once across the sweep, an older one adds nothing', async () => {
diff --git a/backend/integration/workerIndexPlans.test.ts b/backend/integration/workerIndexPlans.test.ts
index 2e315d9e3..cac2148da 100644
--- a/backend/integration/workerIndexPlans.test.ts
+++ b/backend/integration/workerIndexPlans.test.ts
@@ -217,7 +217,7 @@ describe('the operation receipts\' expiry (superTabs/operations.ts)', () => {
})
})
-describe('the usage-receipt sweep\'s stuck check (lib/productUsageReceiptSweep.ts lastOldReceipt)', () => {
+describe('the usage-receipt sweep\'s done check (lib/productUsageReceiptSweep.ts lastOldReceipt)', () => {
// backend/AGENTS.md rule 3: its hourly read while the rule is refused takes one key of the expiry index, never a scan of millions.
it('reads the newest old receipt by one key of the product_usage_expiry index', async () => {
const { lastOldReceipt } = await import('../src/lib/productUsageReceiptSweep.js')
diff --git a/backend/src/lib/productUsageReceiptSweep.ts b/backend/src/lib/productUsageReceiptSweep.ts
index 4c682c6a0..66e6d9dbb 100644
--- a/backend/src/lib/productUsageReceiptSweep.ts
+++ b/backend/src/lib/productUsageReceiptSweep.ts
@@ -18,18 +18,26 @@ import { notAuthorized } from './db/indexes.js'
* again once its receipt is gone. Its receipt write and its totals share one transaction, so with the rule it counts nothing
* and its client retries later. No server's presence is inferred: the rule holds whatever runs. Mongo applies a rule to
* writes from then on only (receipts are never updated), so installing it reads no existing receipt.
+ *
+ * Production's app user may not install the rule (no collMod), and the sweep then removes without it (Ken, 2026-11-04): a
+ * server from before 2026-10-29 is no longer deployed, and only a rollback to one could count a retried report twice.
*/
const COLLECTION = 'product_usage_receipts'
const DAY = 86_400_000
const OLD_EXPIRY_DAYS = 36
-/** About 2,000 a minute, in four deletes with a pause between them: the backlog clears in about a day. While the database
- * refuses the rule, the worker asks again once an hour. */
-export const RECEIPT_SWEEP = { chunk: 500, chunks: 4, pauseMs: 250, everyMs: 60_000, refusedEveryMs: 3_600_000 } as const
+/** One delete of 500 every 30 s, about 1,000 a minute. On production's M20 a delete of 500 old receipts took 12–17 s (cold
+ * index pages, read from disk; 2026-11-04), so the database spends about half its time on them at most, and a pass still
+ * running when the next is due skips it. The ~4M old receipts take about 3 days, against 2026-12-04 by expiry alone. */
+export const RECEIPT_SWEEP = { chunk: 500, chunks: 1, pauseMs: 0, everyMs: 30_000 } as const
/** Every receipt write must carry its life; old receipts, never written again, are not checked. */
export const RECEIPT_RULE = { validator: { $jsonSchema: { required: ['life'] } }, validationLevel: 'strict', validationAction: 'error' } as const
-export interface SweepOptions { chunk: number; chunks: number; pauseMs: number }
+export interface SweepOptions {
+ chunk: number; chunks: number; pauseMs: number
+ /** Removes without installing [RECEIPT_RULE] first: the database user may not (production's, no collMod). */
+ withoutRule?: boolean
+}
export interface SweepPass {
removed: number
/** No old receipt is left that this sweep would ever remove. */
@@ -92,9 +100,10 @@ async function removeChunk(db: PrismaClient, now: Date, chunk: number): Promise<
return answer.n ?? 0
}
-/** One pass: the rule, then at most [chunks] × [chunk] receipts. Safe to repeat, and from two processes at once. */
+/** One pass: the rule unless [SweepOptions.withoutRule], then at most [chunks] × [chunk] receipts. Safe to repeat, and from
+ * two processes at once. */
export async function sweepOldReceipts(db: PrismaClient, now = new Date(), options: SweepOptions = RECEIPT_SWEEP): Promise<SweepPass> {
- await ensureReceiptRule(db)
+ if (!options.withoutRule) await ensureReceiptRule(db)
let removed = 0
for (let i = 0; i < options.chunks; i++) {
if (i) await delay(options.pauseMs)
@@ -105,19 +114,17 @@ export async function sweepOldReceipts(db: PrismaClient, now = new Date(), optio
return { removed, done: !await lastOldReceipt(db, now) }
}
-/** Runs a pass a minute in the singleton worker until nothing is left to remove, then stops; a restart checks once more.
- * While the database refuses the rule, nothing is removed: each hourly check logs an error saying so and when the old receipts
- * expire by themselves (a single warning at start went unseen from 10-29 to 11-03, task #2488), and once none is left the
- * sweep stops, rule or not: Mongo's expiry has done its work. */
-export function startReceiptSweep(db: PrismaClient, settings: Partial<SweepOptions & { everyMs: number; refusedEveryMs: number;
- now: () => Date; log: Pick<typeof logger, 'info' | 'warn' | 'error'> }> = {}): () => void {
- const { everyMs, refusedEveryMs, now, log, ...options } = { ...RECEIPT_SWEEP, now: () => new Date(), log: logger, ...settings }
- let running = false, total = 0, refusedUntil = 0
+/** Runs a pass every [RECEIPT_SWEEP.everyMs] in the singleton worker until nothing is left to remove, then stops; a restart
+ * checks once more. A database user that may not install the rule is asked once: the sweep says so once and goes on without. */
+export function startReceiptSweep(db: PrismaClient, settings: Partial<SweepOptions & { everyMs: number;
+ now: () => Date; log: Pick<typeof logger, 'info' | 'warn'> }> = {}): () => void {
+ const { everyMs, now, log, ...options } = { ...RECEIPT_SWEEP, now: () => new Date(), log: logger, ...settings }
+ let running = false, total = 0, withoutRule = options.withoutRule ?? false
const run = async () => {
- if (running || now().getTime() < refusedUntil) return
+ if (running) return
running = true
try {
- const pass = await sweepOldReceipts(db, now(), options)
+ const pass = await sweepOldReceipts(db, now(), { ...options, withoutRule })
total += pass.removed
if (pass.removed) log.info('[usage-receipts] pass', { removed: pass.removed, removedSinceStart: total })
if (pass.done) {
@@ -126,17 +133,12 @@ export function startReceiptSweep(db: PrismaClient, settings: Partial<SweepOptio
}
} catch (error) {
if (error instanceof ReceiptRuleRefused) {
- refusedUntil = now().getTime() + refusedEveryMs
- const last = await lastOldReceipt(db, now()).catch(() => undefined)
- if (last === null) {
- stop()
- log.info('[usage-receipts] done: no old receipts left', { removedSinceStart: total })
- return
- }
- log.error(`[usage-receipts] stuck: the database user may not install the receipt rule (collMod), so nothing is removed. The old receipts expire by themselves by ${last ? last.toISOString().slice(0, 10) : 'a date it could not read'}; grant collMod to finish sooner. Checking hourly.`)
+ withoutRule = true
+ log.warn('[usage-receipts] the database user may not install the receipt rule (collMod): removing the old receipts without it '
+ + '(Ken, 2026-11-04). A server from before 2026-10-29, were one rolled back, could count a retried report twice.')
return
}
- log.warn('[usage-receipts] pass failed; retrying next minute', { error: error instanceof Error ? error.message : String(error) })
+ log.warn('[usage-receipts] pass failed; retrying next pass', { error: error instanceof Error ? error.message : String(error) })
} finally {
running = false
}