The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.
TypeScript 3d-modelingagent-frameworkagent-harnessai-agentscadclaude-codecodexcoding-agentsdeveloper-toolsesp32game-developmentmcp-servermulti-agentopen-hardwareopencodeopenharnesspcbroboticsterminaltmux
Backend: the usage-receipt sweep removes old receipts without its rule when the database refuses it, one chunk every 30 s (#2912)
0xken158af48019036b
diff --git a/backend/integration/productUsageReceipts.test.ts b/backend/integration/productUsageReceipts.test.tsindex 55112e8e6..aeecf3019 100644--- a/backend/integration/productUsageReceipts.test.ts+++ b/backend/integration/productUsageReceipts.test.ts@@ -200,7 +200,7 @@ it('the worker\'s sweep runs a pass a minute and stops by itself once nothing isexpect(await left()).toEqual(KEPT.filter(id => !id.startsWith('old-2026-10-30')).sort())})-it('a failed pass is logged and the next minute\'s pass carries on', async () => {+it('a failed pass is logged and the next pass carries on', async () => {const lines: string[] = []const log = { info: (message: string) => lines.push(message), warn: (message: string) => lines.push(message) }await db.productUsageReceipt.deleteMany({ where: { id: { startsWith: 'old-2026-10-30' } } })@@ -213,7 +213,7 @@ it('a failed pass is logged and the next minute\'s pass carries on', async () =>try {await vi.waitFor(() => expect(lines.at(-1)).toMatch(/done/), { timeout: 5000 })} finally { stop() }- expect(lines[0]).toMatch(/pass failed; retrying next minute/)+ expect(lines[0]).toMatch(/pass failed; retrying next pass/)expect(await left()).toEqual(KEPT.filter(id => !id.startsWith('old-2026-10-30')).sort())})@@ -233,37 +233,27 @@ function refusingCollMod(refused: () => boolean) {/** Each line the sweep logs, with its level. */const levels = (lines: string[]) => ({ info: (message: string) => lines.push(`info ${message}`), warn: (message: string) => lines.push(`warn ${message}`),error: (message: string) => lines.push(`error ${message}`) })-const STUCK = 'error [usage-receipts] stuck: the database user may not install the receipt rule (collMod), so nothing is removed. The old receipts ' +- 'expire by themselves by 2026-12-04; grant collMod to finish sooner. Checking hourly.'+const WITHOUT_RULE = 'warn [usage-receipts] the database user may not install the receipt rule (collMod): removing the old receipts ' ++ 'without it (Ken, 2026-11-04). A server from before 2026-10-29, were one rolled back, could count a retried report twice.'-it('a database user that may not install the rule gets an error every hourly check, then the sweep runs once it may (prod 10-29, task #2488)', async () => {+it('a database user that may not install the rule asks once, says so once, and removes the old receipts anyway (Ken, 2026-11-04)', async () => {const lines: string[] = []await db.productUsageReceipt.deleteMany({ where: { id: { startsWith: 'old-2026-10-30' } } })- let refused = true, clock = now.getTime()- const { client, tries } = refusingCollMod(() => refused)- const stop = sweep.startReceiptSweep(client, { everyMs: 20, chunk: 50, chunks: 1, pauseMs: 0, now: () => new Date(clock), log: levels(lines) })+ const { client, tries } = refusingCollMod(() => true)+ const stop = sweep.startReceiptSweep(client, { everyMs: 20, chunk: 50, chunks: 1, pauseMs: 0, now: () => now, log: levels(lines) })try {- await vi.waitFor(() => expect(tries.installs).toBe(1), { timeout: 5000 })- // Many minutes' passes later, within the hour: no second try, no second line.- await new Promise(resolve => setTimeout(resolve, 200))- expect(tries.installs).toBe(1)- expect(lines).toEqual([STUCK])- expect((await left()).length).toBe(109)- clock += 3_600_000- await vi.waitFor(() => expect(tries.installs).toBe(2), { timeout: 5000 })- await new Promise(resolve => setTimeout(resolve, 200))- expect(tries.installs).toBe(2)- // Every hour it is stuck says so again: one quiet warning at start was missed for days (#2245).- expect(lines).toEqual([STUCK, STUCK])- // Ken grants collMod: the next hourly check installs the rule and the sweep carries on, without a restart.- refused = false- clock += 3_600_000await vi.waitFor(() => expect(lines.at(-1)).toMatch(/done/), { timeout: 5000 })} finally { stop() }+ expect(tries.installs).toBe(1)+ expect(lines[0]).toBe(WITHOUT_RULE)+ expect(lines.filter(line => line === WITHOUT_RULE)).toHaveLength(1)expect(await left()).toEqual(KEPT.filter(id => !id.startsWith('old-2026-10-30')).sort())+ const [info] = (await db.$runCommandRaw({ listCollections: 1, filter: { name: 'product_usage_receipts' } }) as+ { cursor: { firstBatch: Array<{ options: Record<string, unknown> }> } }).cursor.firstBatch+ expect(info.options).toMatchObject({ validationLevel: 'off' })})-it('a refused sweep stops by itself once its old receipts have expired, and asks for collMod no more (task #2488)', async () => {+it('a refused sweep stops by itself once its old receipts have expired, and asks for collMod no more', async () => {const lines: string[] = []// Mongo's expiry has removed every old receipt the sweep would have: those it never touches (within 5 days) are left.await db.productUsageReceipt.deleteMany({ where: { id: { startsWith: 'old-' }, expiresAt: { gt: at('2026-11-02', 5) } } })@@ -275,8 +265,8 @@ it('a refused sweep stops by itself once its old receipts have expired, and asksclock += 3 * 3_600_000await new Promise(resolve => setTimeout(resolve, 200))} finally { stop() }- expect(lines).toEqual(['info [usage-receipts] done: no old receipts left'])- expect(tries.installs).toBeLessThanOrEqual(1)+ expect(lines).toEqual([WITHOUT_RULE, 'info [usage-receipts] done: no old receipts left'])+ expect(tries.installs).toBe(1)})it('counting stays exact: a report inside the window counts once across the sweep, an older one adds nothing', async () => {diff --git a/backend/integration/workerIndexPlans.test.ts b/backend/integration/workerIndexPlans.test.tsindex 2e315d9e3..cac2148da 100644--- a/backend/integration/workerIndexPlans.test.ts+++ b/backend/integration/workerIndexPlans.test.ts@@ -217,7 +217,7 @@ describe('the operation receipts\' expiry (superTabs/operations.ts)', () => {})})-describe('the usage-receipt sweep\'s stuck check (lib/productUsageReceiptSweep.ts lastOldReceipt)', () => {+describe('the usage-receipt sweep\'s done check (lib/productUsageReceiptSweep.ts lastOldReceipt)', () => {// backend/AGENTS.md rule 3: its hourly read while the rule is refused takes one key of the expiry index, never a scan of millions.it('reads the newest old receipt by one key of the product_usage_expiry index', async () => {const { lastOldReceipt } = await import('../src/lib/productUsageReceiptSweep.js')diff --git a/backend/src/lib/productUsageReceiptSweep.ts b/backend/src/lib/productUsageReceiptSweep.tsindex 4c682c6a0..66e6d9dbb 100644--- a/backend/src/lib/productUsageReceiptSweep.ts+++ b/backend/src/lib/productUsageReceiptSweep.ts@@ -18,18 +18,26 @@ import { notAuthorized } from './db/indexes.js'* again once its receipt is gone. Its receipt write and its totals share one transaction, so with the rule it counts nothing* and its client retries later. No server's presence is inferred: the rule holds whatever runs. Mongo applies a rule to* writes from then on only (receipts are never updated), so installing it reads no existing receipt.+ *+ * Production's app user may not install the rule (no collMod), and the sweep then removes without it (Ken, 2026-11-04): a+ * server from before 2026-10-29 is no longer deployed, and only a rollback to one could count a retried report twice.*/const COLLECTION = 'product_usage_receipts'const DAY = 86_400_000const OLD_EXPIRY_DAYS = 36-/** About 2,000 a minute, in four deletes with a pause between them: the backlog clears in about a day. While the database- * refuses the rule, the worker asks again once an hour. */-export const RECEIPT_SWEEP = { chunk: 500, chunks: 4, pauseMs: 250, everyMs: 60_000, refusedEveryMs: 3_600_000 } as const+/** One delete of 500 every 30 s, about 1,000 a minute. On production's M20 a delete of 500 old receipts took 12–17 s (cold+ * index pages, read from disk; 2026-11-04), so the database spends about half its time on them at most, and a pass still+ * running when the next is due skips it. The ~4M old receipts take about 3 days, against 2026-12-04 by expiry alone. */+export const RECEIPT_SWEEP = { chunk: 500, chunks: 1, pauseMs: 0, everyMs: 30_000 } as const/** Every receipt write must carry its life; old receipts, never written again, are not checked. */export const RECEIPT_RULE = { validator: { $jsonSchema: { required: ['life'] } }, validationLevel: 'strict', validationAction: 'error' } as const-export interface SweepOptions { chunk: number; chunks: number; pauseMs: number }+export interface SweepOptions {+ chunk: number; chunks: number; pauseMs: number+ /** Removes without installing [RECEIPT_RULE] first: the database user may not (production's, no collMod). */+ withoutRule?: boolean+}export interface SweepPass {removed: number/** No old receipt is left that this sweep would ever remove. */@@ -92,9 +100,10 @@ async function removeChunk(db: PrismaClient, now: Date, chunk: number): Promise<return answer.n ?? 0}-/** One pass: the rule, then at most [chunks] × [chunk] receipts. Safe to repeat, and from two processes at once. */+/** One pass: the rule unless [SweepOptions.withoutRule], then at most [chunks] × [chunk] receipts. Safe to repeat, and from+ * two processes at once. */export async function sweepOldReceipts(db: PrismaClient, now = new Date(), options: SweepOptions = RECEIPT_SWEEP): Promise<SweepPass> {- await ensureReceiptRule(db)+ if (!options.withoutRule) await ensureReceiptRule(db)let removed = 0for (let i = 0; i < options.chunks; i++) {if (i) await delay(options.pauseMs)@@ -105,19 +114,17 @@ export async function sweepOldReceipts(db: PrismaClient, now = new Date(), optioreturn { removed, done: !await lastOldReceipt(db, now) }}-/** Runs a pass a minute in the singleton worker until nothing is left to remove, then stops; a restart checks once more.- * While the database refuses the rule, nothing is removed: each hourly check logs an error saying so and when the old receipts- * expire by themselves (a single warning at start went unseen from 10-29 to 11-03, task #2488), and once none is left the- * sweep stops, rule or not: Mongo's expiry has done its work. */-export function startReceiptSweep(db: PrismaClient, settings: Partial<SweepOptions & { everyMs: number; refusedEveryMs: number;- now: () => Date; log: Pick<typeof logger, 'info' | 'warn' | 'error'> }> = {}): () => void {- const { everyMs, refusedEveryMs, now, log, ...options } = { ...RECEIPT_SWEEP, now: () => new Date(), log: logger, ...settings }- let running = false, total = 0, refusedUntil = 0+/** Runs a pass every [RECEIPT_SWEEP.everyMs] in the singleton worker until nothing is left to remove, then stops; a restart+ * checks once more. A database user that may not install the rule is asked once: the sweep says so once and goes on without. */+export function startReceiptSweep(db: PrismaClient, settings: Partial<SweepOptions & { everyMs: number;+ now: () => Date; log: Pick<typeof logger, 'info' | 'warn'> }> = {}): () => void {+ const { everyMs, now, log, ...options } = { ...RECEIPT_SWEEP, now: () => new Date(), log: logger, ...settings }+ let running = false, total = 0, withoutRule = options.withoutRule ?? falseconst run = async () => {- if (running || now().getTime() < refusedUntil) return+ if (running) returnrunning = truetry {- const pass = await sweepOldReceipts(db, now(), options)+ const pass = await sweepOldReceipts(db, now(), { ...options, withoutRule })total += pass.removedif (pass.removed) log.info('[usage-receipts] pass', { removed: pass.removed, removedSinceStart: total })if (pass.done) {@@ -126,17 +133,12 @@ export function startReceiptSweep(db: PrismaClient, settings: Partial<SweepOptio}} catch (error) {if (error instanceof ReceiptRuleRefused) {- refusedUntil = now().getTime() + refusedEveryMs- const last = await lastOldReceipt(db, now()).catch(() => undefined)- if (last === null) {- stop()- log.info('[usage-receipts] done: no old receipts left', { removedSinceStart: total })- return- }- log.error(`[usage-receipts] stuck: the database user may not install the receipt rule (collMod), so nothing is removed. The old receipts expire by themselves by ${last ? last.toISOString().slice(0, 10) : 'a date it could not read'}; grant collMod to finish sooner. Checking hourly.`)+ withoutRule = true+ log.warn('[usage-receipts] the database user may not install the receipt rule (collMod): removing the old receipts without it '+ + '(Ken, 2026-11-04). A server from before 2026-10-29, were one rolled back, could count a retried report twice.')return}- log.warn('[usage-receipts] pass failed; retrying next minute', { error: error instanceof Error ? error.message : String(error) })+ log.warn('[usage-receipts] pass failed; retrying next pass', { error: error instanceof Error ? error.message : String(error) })} finally {running = false}