Skip to content

The ultimate harness for coding agents and beyond. All your agents. All your machines. One command center. Start with code, then follow your curiosity and build across disciplines: CAD, circuits, robots, games and music.

19.5 KB
Raw Blame
import { randomUUID } from 'node:crypto'
import { execFile } from 'node:child_process'
import { promisify } from 'node:util'
import { join } from 'node:path'
import { tmpdir } from 'node:os'
import { afterAll, beforeAll, beforeEach, expect, it, vi } from 'vitest'
import { MongoMemoryReplSet } from 'mongodb-memory-server-core'
import { PrismaClient } from '@prisma/client'
import { oldServerIngest } from './oldUsageServer.test-support.js'

// A fresh loopback replica set, created here; nothing reads an existing database. Receipts lived 36 days until 2026-10-29
// (about 4M of them in production) and live 4 now. Here the shorter life was released on 30 October and the clock reads
// 2 November, 03:00 UTC: every receipt below is seeded by this file.
let replica: MongoMemoryReplSet
let db: PrismaClient
let usage: typeof import('../src/lib/productUsage.js')
let sweep: typeof import('../src/lib/productUsageReceiptSweep.js')

beforeAll(async () => {
  replica = await MongoMemoryReplSet.create({ binary: { version: '7.0.24', downloadDir: join(tmpdir(), 'harness-super-tab-mongodb-binaries') },
    replSet: { count: 1, ip: '127.0.0.1', storageEngine: 'wiredTiger' } })
  const uri = replica.getUri(`product_usage_receipts_${randomUUID().replaceAll('-', '_')}`)
  await promisify(execFile)(join(process.cwd(), 'node_modules/.bin/prisma'), ['db', 'push', '--skip-generate'], {
    env: { ...process.env, DATABASE_URL: uri }, timeout: 60000,
  })
  process.env.DATABASE_URL = uri
  vi.resetModules()
  db = (await import('../src/lib/prisma.js')).prisma
  usage = await import('../src/lib/productUsage.js')
  sweep = await import('../src/lib/productUsageReceiptSweep.js')
}, 120000)
afterAll(async () => {
  await db?.$disconnect()
  await replica?.stop()
})

const DAY = 86_400_000
const now = new Date('2026-11-02T03:00:00Z')
const at = (day: string, days: number) => new Date(Date.parse(day) + days * DAY)
/** Every UTC day from [first] to [last], both included. */
const days = (first: string, last: string) => Array.from({ length: (Date.parse(last) - Date.parse(first)) / DAY + 1 },
  (_, i) => at(first, i).toISOString().slice(0, 10))
const ids = (style: string, day: string) => [0, 1, 2].map(n => `${style}-${day}-${n}`)

// Three receipts a report day. Old servers wrote day + 36 up to the release (30 October included), new ones day + 4.
const OLD_DAYS = days('2026-09-28', '2026-10-30')
const NEW_DAYS = days('2026-10-30', '2026-11-02')
// The sweep removes the old receipts whose 4-day life is over (report days 3 to 29 October), apart from those Mongo's expiry
// removes within 5 days anyway (day + 36 up to 7 November). The 30th's new life ends on 3 November, after this clock.
const KEPT = [...days('2026-09-28', '2026-10-02'), '2026-10-30'].flatMap(day => ids('old', day))
  .concat(NEW_DAYS.flatMap(day => ids('new', day)), ['expired-old-2026-09-25'])
const TOTALS = [{ dayUtc: new Date('2026-10-20'), metric: 'turns_started', value: 41 }, { dayUtc: new Date('2026-11-01'), metric: 'turns_started', value: 7 }]

beforeEach(async () => {
  // Totals a test left waiting in memory must not land in the next test's database.
  await usage.flushRelayUsage().catch(() => {})
  // The database as it is before the release: no receipt rule yet.
  await db.$runCommandRaw({ collMod: 'product_usage_receipts', validator: {}, validationLevel: 'off' })
  await db.productUsageReceipt.deleteMany()
  await db.productUsageTotal.deleteMany()
  await db.productUsageReceipt.createMany({ data: [
    ...OLD_DAYS.flatMap(day => ids('old', day).map(id => ({ id, expiresAt: at(day, 36) }))),
    ...NEW_DAYS.flatMap(day => ids('new', day).map(id => ({ id, expiresAt: at(day, 4), life: 4 }))),
    // Expired already (1 November); Mongo's expiry removes it, not the cleanup.
    { id: 'expired-old-2026-09-25', expiresAt: at('2026-09-25', 36) },
  ] })
  await db.productUsageTotal.createMany({ data: TOTALS })
})

/** Resolves once [ready] holds, checking every 10 ms. */
async function until(ready: () => Promise<boolean>): Promise<void> {
  while (!await ready()) await new Promise(resolve => setTimeout(resolve, 10))
}
/** How many installs of a rule on the receipts wait for a lock now. */
async function installsWaiting(): Promise<number> {
  const admin = new PrismaClient({ datasources: { db: { url: replica.getUri('admin') } } })
  try {
    const ops = await admin.$runCommandRaw({ aggregate: 1, pipeline: [{ $currentOp: {} },
      { $match: { 'command.collMod': 'product_usage_receipts', waitingForLock: true } }], cursor: {} }) as { cursor: { firstBatch: unknown[] } }
    return ops.cursor.firstBatch.length
  } finally { await admin.$disconnect() }
}
const deferred = () => { let resolve!: () => void; const promise = new Promise<void>(r => { resolve = r }); return { promise, resolve } }
const pass = (at = now) => sweep.sweepOldReceipts(db, at, { chunk: 10, chunks: 4, pauseMs: 30 })
const left = async () => (await db.productUsageReceipt.findMany({ select: { id: true } })).map(r => r.id).sort()
const totals = async () => db.productUsageTotal.findMany({ select: { dayUtc: true, metric: true, value: true }, orderBy: { dayUtc: 'asc' } })

it('removes only old receipts outside the new life, a few chunks a pass with a pause, then nothing', async () => {
  const started = performance.now()
  expect(await pass()).toEqual({ removed: 40, done: false })
  expect(performance.now() - started).toBeGreaterThanOrEqual(3 * 30)
  expect(await pass()).toEqual({ removed: 40, done: false })
  expect(await pass()).toEqual({ removed: 1, done: false })
  expect(await left()).toEqual(KEPT.sort())
  // The 30 October's old receipts are still inside their new life (to 3 November): kept, and the sweep goes on.
  expect(await pass()).toEqual({ removed: 0, done: false })
  const later = new Date('2026-11-03T03:00:00Z')
  expect(await pass(later)).toEqual({ removed: 3, done: true })
  expect(await pass(later)).toEqual({ removed: 0, done: true })
  expect(await left()).toEqual(KEPT.filter(id => !id.startsWith('old-2026-10-30')).sort())
  expect(await totals()).toEqual(TOTALS)
})

it('sends commands to the receipts collection only', async () => {
  await db.$runCommandRaw({ profile: 2 })
  const since = new Date()
  expect(await pass()).toMatchObject({ removed: 40 })
  await db.$runCommandRaw({ profile: 0 })
  const answer = await db.$runCommandRaw({ find: 'system.profile', filter: { ts: { $gt: { $date: since.toISOString() } } },
    projection: { ns: 1 }, batchSize: 100_000, singleBatch: true }) as { cursor: { firstBatch: Array<{ ns: string }> } }
  const collections = new Set(answer.cursor.firstBatch.map(entry => entry.ns.split('.').slice(1).join('.')))
  expect([...collections].filter(name => name !== 'system.profile' && name !== '$cmd')).toEqual(['product_usage_receipts'])
})

it('a chunk is as large as asked, and its delete reads only the receipts it removes (prod 11-04: 101 a chunk, ~4M examined each)', async () => {
  // Mongo's first batch holds 101 documents unless asked for more, and a delete matching on both _id and the expiry range was
  // planned on the expiry index: it examined every old receipt to remove a few hundred.
  await db.productUsageReceipt.createMany({ data: Array.from({ length: 400 }, (_, i) => ({ id: `bulk-${i}`, expiresAt: at('2026-10-10', 36) })) })
  await db.$runCommandRaw({ profile: 2 })
  const since = new Date()
  expect(await sweep.sweepOldReceipts(db, now, { chunk: 300, chunks: 1, pauseMs: 0 })).toMatchObject({ removed: 300 })
  await db.$runCommandRaw({ profile: 0 })
  const answer = await db.$runCommandRaw({ find: 'system.profile', filter: { ts: { $gt: { $date: since.toISOString() } }, op: 'remove' },
    projection: { docsExamined: 1, planSummary: 1 }, batchSize: 100_000, singleBatch: true }) as
    { cursor: { firstBatch: Array<{ docsExamined: number; planSummary: string }> } }
  expect(answer.cursor.firstBatch.length).toBeGreaterThan(0)
  for (const op of answer.cursor.firstBatch) {
    expect(op.planSummary).toMatch(/_id/)
    expect(op.docsExamined).toBeLessThanOrEqual(300)
  }
})

const late = (day = '2026-10-29') => ({ id: randomUUID(), day, counters: { turns_ended: 2 } })
const turnsEnded = async (day: string) =>
  (await db.productUsageTotal.findUnique({ where: { dayUtc_metric: { dayUtc: new Date(day), metric: 'turns_ended' } } }))?.value
const drain = async (at = now) => { while ((await pass(at)).removed) { /* the next chunks */ } }
const account = '6a0000000000000000000c01'
// A report's totals wait in memory until the minute's flush; the flush is part of counting it.
const signedIn = async (report: ReturnType<typeof late>, at = now) => {
  const answer = await usage.ingestProductUsage(account, { schemaVersion: 1, revision: 0,
    ...report, context: { surface: 'cli', release: '0.4.92', platform: 'macos', channel: 'production' } }, at)
  await usage.flushRelayUsage()
  return answer
}

it('an old server still running cannot count a late report twice across the sweep (review of c20be500d)', async () => {
  // 2 November 00:01: no report for today yet, and an old server takes a 29 October report.
  const midnight = new Date('2026-11-02T00:01:00Z'), report = late()
  expect(await oldServerIngest(db, report, midnight)).toEqual({ accepted: true })
  expect(await turnsEnded(report.day)).toBe(2)
  await drain(midnight)
  expect(await db.productUsageReceipt.findUnique({ where: { id: report.id } })).toBeNull()
  // The client retries through the old server: it may no longer write a receipt, so nothing is added.
  await expect(oldServerIngest(db, report, midnight)).rejects.toThrow(/validation/i)
  expect(await signedIn(report, midnight)).toEqual({ accepted: false, reason: 'EXPIRED' })
  expect(await turnsEnded(report.day)).toBe(2)
})

it('removes nothing until every receipt write must carry the new life', async () => {
  const refusing = new Proxy(db, { get: (target, key) => key === '$runCommandRaw'
    ? (command: Record<string, unknown>) => 'collMod' in command ? Promise.reject(new Error('not authorized')) : target.$runCommandRaw(command as never)
    : Reflect.get(target, key) })
  await expect(sweep.sweepOldReceipts(refusing, now, { chunk: 10, chunks: 4, pauseMs: 0 })).rejects.toThrow('not authorized')
  expect((await left()).length).toBe(112)
  await pass()
  const [info] = (await db.$runCommandRaw({ listCollections: 1, filter: { name: 'product_usage_receipts' } }) as
    { cursor: { firstBatch: Array<{ options: { validator?: unknown; validationLevel?: string } }> } }).cursor.firstBatch
  expect(info.options).toMatchObject({ validator: { $jsonSchema: { required: ['life'] } }, validationLevel: 'strict', validationAction: 'error' })
})

it('mixed versions count a report inside the window once, whichever server takes each copy', async () => {
  // Before the sweep: the old server takes it first; the new server sees its receipt.
  const first = late('2026-11-01')
  expect(await oldServerIngest(db, first, now)).toEqual({ accepted: true })
  expect(await signedIn(first)).toEqual({ accepted: true })
  await drain()
  expect(await db.productUsageReceipt.findUnique({ where: { id: first.id } })).not.toBeNull()
  expect(await signedIn(first)).toEqual({ accepted: true })
  // After it: the old server is refused, the new server counts it, and the old server then sees the new receipt.
  const second = { ...late('2026-11-01'), counters: { turns_ended: 5 } }
  await expect(oldServerIngest(db, second, now)).rejects.toThrow(/validation/i)
  expect(await signedIn(second)).toEqual({ accepted: true })
  expect(await oldServerIngest(db, second, now)).toEqual({ accepted: true })
  expect(await turnsEnded('2026-11-01')).toBe(7)
})

it('an old server between its receipt lookup and its write while the rule goes up counts at most once', async () => {
  const report = late(), lookedUp = deferred(), proceed = deferred()
  const old = oldServerIngest(db, report, now, async () => { lookedUp.resolve(); await proceed.promise })
  await lookedUp.promise
  // The rule's install queues behind the old server's open transaction, which then writes: it commits under the old rule,
  // or Mongo ends it for the lock it now waits on. Either way, nothing after the install is written without a life.
  let settled = false
  const sweeping = pass().finally(() => { settled = true })
  await until(async () => settled || (await installsWaiting()) > 0)
  expect(settled).toBe(false)
  proceed.resolve()
  const [first] = await Promise.allSettled([old, sweeping])
  await drain()
  await Promise.allSettled([oldServerIngest(db, report, now), oldServerIngest(db, report, now)])
  expect(await turnsEnded(report.day)).toBe(first.status === 'fulfilled' ? 2 : undefined)
  await expect(oldServerIngest(db, late(), now)).rejects.toThrow(/validation/i)
})

it('two workers at once (a rolling deploy) still remove each receipt once and nothing else', async () => {
  const passes = await Promise.all([pass(), pass(), pass()])
  expect(passes.map(p => p.done)).toEqual([false, false, false])
  while ((await pass()).removed) { /* the rest */ }
  expect(await left()).toEqual(KEPT.sort())
  expect(await totals()).toEqual(TOTALS)
})

it('the worker\'s sweep runs a pass a minute and stops by itself once nothing is left', async () => {
  const lines: string[] = []
  const log = { info: (message: string) => lines.push(message), warn: (message: string) => lines.push(message) }
  await db.productUsageReceipt.deleteMany({ where: { id: { startsWith: 'old-2026-10-30' } } })
  const stop = sweep.startReceiptSweep(db, { everyMs: 20, chunk: 50, chunks: 1, pauseMs: 0, now: () => now, log })
  try {
    await vi.waitFor(() => expect(lines.at(-1)).toMatch(/done/), { timeout: 5000 })
    const settled = lines.length
    await new Promise(resolve => setTimeout(resolve, 100))
    expect(lines.length).toBe(settled)
  } finally { stop() }
  expect(lines.filter(line => line.includes('pass'))).toHaveLength(2)
  expect(await left()).toEqual(KEPT.filter(id => !id.startsWith('old-2026-10-30')).sort())
})

it('a failed pass is logged and the next pass carries on', async () => {
  const lines: string[] = []
  const log = { info: (message: string) => lines.push(message), warn: (message: string) => lines.push(message) }
  await db.productUsageReceipt.deleteMany({ where: { id: { startsWith: 'old-2026-10-30' } } })
  let calls = 0
  // The database refuses the first command only, as a lost connection or a primary stepping down would.
  const flaky = new Proxy(db, { get: (target, key) => key === '$runCommandRaw'
    ? (command: never) => calls++ ? target.$runCommandRaw(command) : Promise.reject(new Error('connection closed'))
    : Reflect.get(target, key) })
  const stop = sweep.startReceiptSweep(flaky, { everyMs: 20, chunk: 100, chunks: 1, pauseMs: 0, now: () => now, log })
  try {
    await vi.waitFor(() => expect(lines.at(-1)).toMatch(/done/), { timeout: 5000 })
  } finally { stop() }
  expect(lines[0]).toMatch(/pass failed; retrying next pass/)
  expect(await left()).toEqual(KEPT.filter(id => !id.startsWith('old-2026-10-30')).sort())
})

/** A client that refuses collMod as Mongo refuses production's app user (code 13) while [refused] says so, counting the tries. */
function refusingCollMod(refused: () => boolean) {
  const tries = { installs: 0 }
  const client = new Proxy(db, { get: (target, key) => key === '$runCommandRaw'
    ? (command: Record<string, unknown>) => {
      if (!('collMod' in command)) return target.$runCommandRaw(command as never)
      tries.installs++
      return refused() ? Promise.reject(new Error('Raw query failed. Code: `13`. Message: `not authorized on harness to execute command { collMod: "product_usage_receipts" }`'))
        : target.$runCommandRaw(command as never)
    }
    : Reflect.get(target, key) })
  return { client, tries }
}
/** Each line the sweep logs, with its level. */
const levels = (lines: string[]) => ({ info: (message: string) => lines.push(`info ${message}`), warn: (message: string) => lines.push(`warn ${message}`),
  error: (message: string) => lines.push(`error ${message}`) })
const WITHOUT_RULE = 'warn [usage-receipts] the database user may not install the receipt rule (collMod): removing the old receipts ' +
  'without it (Ken, 2026-11-04). A server from before 2026-10-29, were one rolled back, could count a retried report twice.'

it('a database user that may not install the rule asks once, says so once, and removes the old receipts anyway (Ken, 2026-11-04)', async () => {
  const lines: string[] = []
  await db.productUsageReceipt.deleteMany({ where: { id: { startsWith: 'old-2026-10-30' } } })
  const { client, tries } = refusingCollMod(() => true)
  const stop = sweep.startReceiptSweep(client, { everyMs: 20, chunk: 50, chunks: 1, pauseMs: 0, now: () => now, log: levels(lines) })
  try {
    await vi.waitFor(() => expect(lines.at(-1)).toMatch(/done/), { timeout: 5000 })
  } finally { stop() }
  expect(tries.installs).toBe(1)
  expect(lines[0]).toBe(WITHOUT_RULE)
  expect(lines.filter(line => line === WITHOUT_RULE)).toHaveLength(1)
  expect(await left()).toEqual(KEPT.filter(id => !id.startsWith('old-2026-10-30')).sort())
  const [info] = (await db.$runCommandRaw({ listCollections: 1, filter: { name: 'product_usage_receipts' } }) as
    { cursor: { firstBatch: Array<{ options: Record<string, unknown> }> } }).cursor.firstBatch
  expect(info.options).toMatchObject({ validationLevel: 'off' })
})

it('a refused sweep stops by itself once its old receipts have expired, and asks for collMod no more', async () => {
  const lines: string[] = []
  // Mongo's expiry has removed every old receipt the sweep would have: those it never touches (within 5 days) are left.
  await db.productUsageReceipt.deleteMany({ where: { id: { startsWith: 'old-' }, expiresAt: { gt: at('2026-11-02', 5) } } })
  let clock = now.getTime()
  const { client, tries } = refusingCollMod(() => true)
  const stop = sweep.startReceiptSweep(client, { everyMs: 20, chunk: 50, chunks: 1, pauseMs: 0, now: () => new Date(clock), log: levels(lines) })
  try {
    await vi.waitFor(() => expect(lines.at(-1)).toMatch(/done/), { timeout: 5000 })
    clock += 3 * 3_600_000
    await new Promise(resolve => setTimeout(resolve, 200))
  } finally { stop() }
  expect(lines).toEqual([WITHOUT_RULE, 'info [usage-receipts] done: no old receipts left'])
  expect(tries.installs).toBe(1)
})

it('counting stays exact: a report inside the window counts once across the sweep, an older one adds nothing', async () => {
  const context = { surface: 'cli', release: '0.4.92', platform: 'macos', channel: 'production' } as const
  const report = { schemaVersion: 1 as const, revision: 0, id: randomUUID(), day: '2026-11-01', context, counters: { turns_ended: 2 } }
  const turnsEnded = async () => (await db.productUsageTotal.findUnique({ where: { dayUtc_metric: { dayUtc: new Date('2026-11-01'), metric: 'turns_ended' } } }))?.value
  expect(await usage.ingestProductUsage('6a0000000000000000000c01', report, now)).toEqual({ accepted: true })
  expect(await db.productUsageReceipt.findUnique({ where: { id: report.id } })).toEqual({ id: report.id, expiresAt: new Date('2026-11-05'), life: 4 })
  while ((await pass()).removed) { /* every old receipt outside the new life */ }
  // The rule is up now: the flush writing the report's totals must carry its life too, or it would count nothing.
  await usage.flushRelayUsage()
  expect(await usage.ingestProductUsage('6a0000000000000000000c01', report, now)).toEqual({ accepted: true })
  await usage.flushRelayUsage()
  expect(await turnsEnded()).toBe(2)

  const late = { ...report, id: randomUUID(), day: '2026-10-30' }
  expect(await usage.ingestProductUsage('6a0000000000000000000c01', late, now)).toEqual({ accepted: false, reason: 'EXPIRED' })
  expect(await db.productUsageReceipt.findUnique({ where: { id: late.id } })).toBeNull()
  expect(await db.productUsageTotal.findMany({ where: { dayUtc: new Date('2026-10-30') } })).toEqual([])
}, 60000)