Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
perf(relay): pause drained settled-credential reaps for 5 minutes (STA-7899) (#36144)
Neil6f880fbcbd10
diff --git a/cloud/apps/relay/src/credential-store-cleanup.test.ts b/cloud/apps/relay/src/credential-store-cleanup.test.tsindex ef5548c2613..5feff4a4713 100644--- a/cloud/apps/relay/src/credential-store-cleanup.test.ts+++ b/cloud/apps/relay/src/credential-store-cleanup.test.ts@@ -318,6 +318,26 @@ describe('credential cleanup invite reaper', () => {await database.close()})+ it('skips drained invite and authorization reaps until the pause ends', async () => {+ const database = await openInMemoryRelayDatabase()+ let now = NOW+ const store = new RelayCredentialStore(database, () => now)+ await store.cleanup()+ await insertInvite(database, { token: 'stale', state: 'consumed', updatedAt: NOW - 8 * DAY_MS })+ await insertDirectAuthorization(database, { id: 'stale', deadline: NOW, consumedAt: NOW - 2 * DAY_MS })++ now = NOW + 60_000+ await store.cleanup()+ expect(await remainingTokens(database)).toEqual(['stale'])+ expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual(['stale'])++ now = NOW + 6 * 60_000+ await store.cleanup()+ expect(await remainingTokens(database)).toEqual([])+ expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual([])+ await database.close()+ })+it('reaps confirm results and audit events only past their retention windows', async () => {const database = await openInMemoryRelayDatabase()const store = new RelayCredentialStore(database, () => NOW)diff --git a/cloud/apps/relay/src/credential-store.ts b/cloud/apps/relay/src/credential-store.tsindex 5447431072c..92c36c026f1 100644--- a/cloud/apps/relay/src/credential-store.ts+++ b/cloud/apps/relay/src/credential-store.ts@@ -6,7 +6,7 @@ import {type DeviceResumeConfirmed} from '@orca-cloud/relay-contract'import type { RelayDatabase, SqlRow } from './database.js'-import { HeapWindowReaper } from './heap-window-reaper.js'+import { changes, HeapWindowReaper } from './heap-window-reaper.js'import { ResumeConfirmedAuditBuffer } from './resume-confirmed-audit-buffer.js'const CREDENTIAL_GRACE_MS = 24 * 60 * 60 * 1000@@ -23,6 +23,8 @@ export const AUDIT_EVENT_RETENTION_MS = 90 * 24 * 60 * 60 * 1000// Bounded so one cycle cannot hold row locks or grow WAL without limit; the backlog drains over// however many cycles it takes.const REAP_BATCH_ROWS = 5000+// Unindexed, a drained reap rescans the whole heap; retention is days, so minutes of reap lag are harmless.+const UNINDEXED_REAP_PAUSE_MS = 5 * 60 * 1000// At ~14 rows/page and ~9 ticks/min, budget reaps ~3M rows/day (~2–3 days for 7.4M backlog); empty walks read ~1MB/tick.const CONFIRM_RESULT_REAP_BUDGET = {pagesPerStatement: 16,@@ -152,6 +154,7 @@ export class RelayCredentialStore {'committed_at <= ?',CONFIRM_RESULT_REAP_BUDGET)+ private readonly unindexedReapResumesAt = new Map<string, number>()// Only memory confirms add to it, so its flush timer never starts on a director.readonly resumeAudit: ResumeConfirmedAuditBuffer@@ -772,10 +775,11 @@ export class RelayCredentialStore {// Idempotent reaping stays outside transition transaction or locks last across unrelated batch deletes.private async reapSettledCredentials(now: number): Promise<void> {- await this.reapBatch(+ await this.reapUnindexed('relay_invites','state IN (?, ?, ?) AND updated_at <= ?',- ['expired', 'consumed', 'invalidated', now - TERMINAL_INVITE_RETENTION_MS]+ ['expired', 'consumed', 'invalidated', now - TERMINAL_INVITE_RETENTION_MS],+ now)// Use indexed active/deadline range to avoid scanning 1.5GB drained heap; both readers reject expired deadlines regardless of flag, so one-day-old bases are unusable.await this.reapBatch('relay_connection_bases', 'active = ? AND deadline <= ?', [@@ -784,10 +788,11 @@ export class RelayCredentialStore {])// consumed_at, not deadline: consumption is what settles this row, and it can happen well// before the deadline, so measuring from it retains the row for the full window either way.- await this.reapBatch(+ await this.reapUnindexed('relay_direct_authorizations','consumed_at IS NOT NULL AND consumed_at <= ?',- [now - INACTIVE_AUTHORIZATION_RETENTION_MS]+ [now - INACTIVE_AUTHORIZATION_RETENTION_MS],+ now)await this.confirmResultReaper.reap(this.database, [now - CONFIRM_RESULT_RETENTION_MS])// Ordered so the plan walks relay_audit_events_at from its oldest entry: unordered, a LIMIT@@ -795,23 +800,40 @@ export class RelayCredentialStore {await this.reapBatch('relay_audit_events', 'at <= ?', [now - AUDIT_EVENT_RETENTION_MS], 'at')}+ // Paused before the delete, so an overlapping or failed reap waits too; only a full batch (a backlog) clears it.+ private async reapUnindexed(+ table: string,+ predicate: string,+ params: unknown[],+ now: number+ ): Promise<void> {+ if (now < (this.unindexedReapResumesAt.get(table) ?? 0)) {+ return+ }+ this.unindexedReapResumesAt.set(table, now + UNINDEXED_REAP_PAUSE_MS)+ if ((await this.reapBatch(table, predicate, params)) >= REAP_BATCH_ROWS) {+ this.unindexedReapResumesAt.delete(table)+ }+ }+// Delete located physical ctid/rowid; PostgreSQL ANY(TID array) guarantees TID scan, while IN can hash-join a full heap scan.private async reapBatch(table: string,predicate: string,params: unknown[],orderBy?: string- ): Promise<void> {+ ): Promise<number> {const sqlite = this.database.dialect === 'sqlite'const address = sqlite ? 'rowid' : 'ctid'const order = orderBy ? `ORDER BY ${orderBy} ` : ''const batch = `SELECT ${address} FROM ${table} WHERE ${predicate} ${order}LIMIT ${REAP_BATCH_ROWS}`- await this.database.query(+ const rows = await this.database.query(sqlite? `DELETE FROM ${table} WHERE rowid IN (${batch})`: `DELETE FROM ${table} WHERE ctid = ANY(ARRAY(${batch}))`,params)+ return changes(rows)}private async installStatusWith(diff --git a/cloud/apps/relay/src/heap-window-reaper.ts b/cloud/apps/relay/src/heap-window-reaper.tsindex 16829dc3f4d..1c78fdc9860 100644--- a/cloud/apps/relay/src/heap-window-reaper.ts+++ b/cloud/apps/relay/src/heap-window-reaper.ts@@ -80,6 +80,6 @@ async function heapPages(database: RelayDatabase, table: string): Promise<numberreturn Number(row?.pages ?? 0)}-function changes(rows: SqlRow[]): number {+export function changes(rows: SqlRow[]): number {return Number(rows[0]?.changes ?? 0)}