Skip to content

armgit / orca

Public Imported from GitHub
0

Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.

perf(relay): pause drained settled-credential reaps for 5 minutes (STA-7899) (#36144)

Neil6f880fbcbd10
diff --git a/cloud/apps/relay/src/credential-store-cleanup.test.ts b/cloud/apps/relay/src/credential-store-cleanup.test.ts
index ef5548c2613..5feff4a4713 100644
--- a/cloud/apps/relay/src/credential-store-cleanup.test.ts
+++ b/cloud/apps/relay/src/credential-store-cleanup.test.ts
@@ -318,6 +318,26 @@ describe('credential cleanup invite reaper', () => {
await database.close()
})
+ it('skips drained invite and authorization reaps until the pause ends', async () => {
+ const database = await openInMemoryRelayDatabase()
+ let now = NOW
+ const store = new RelayCredentialStore(database, () => now)
+ await store.cleanup()
+ await insertInvite(database, { token: 'stale', state: 'consumed', updatedAt: NOW - 8 * DAY_MS })
+ await insertDirectAuthorization(database, { id: 'stale', deadline: NOW, consumedAt: NOW - 2 * DAY_MS })
+
+ now = NOW + 60_000
+ await store.cleanup()
+ expect(await remainingTokens(database)).toEqual(['stale'])
+ expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual(['stale'])
+
+ now = NOW + 6 * 60_000
+ await store.cleanup()
+ expect(await remainingTokens(database)).toEqual([])
+ expect(await remainingIds(database, 'relay_direct_authorizations', 'direct_auth_id')).toEqual([])
+ await database.close()
+ })
+
it('reaps confirm results and audit events only past their retention windows', async () => {
const database = await openInMemoryRelayDatabase()
const store = new RelayCredentialStore(database, () => NOW)
diff --git a/cloud/apps/relay/src/credential-store.ts b/cloud/apps/relay/src/credential-store.ts
index 5447431072c..92c36c026f1 100644
--- a/cloud/apps/relay/src/credential-store.ts
+++ b/cloud/apps/relay/src/credential-store.ts
@@ -6,7 +6,7 @@ import {
type DeviceResumeConfirmed
} from '@orca-cloud/relay-contract'
import type { RelayDatabase, SqlRow } from './database.js'
-import { HeapWindowReaper } from './heap-window-reaper.js'
+import { changes, HeapWindowReaper } from './heap-window-reaper.js'
import { ResumeConfirmedAuditBuffer } from './resume-confirmed-audit-buffer.js'
const CREDENTIAL_GRACE_MS = 24 * 60 * 60 * 1000
@@ -23,6 +23,8 @@ export const AUDIT_EVENT_RETENTION_MS = 90 * 24 * 60 * 60 * 1000
// Bounded so one cycle cannot hold row locks or grow WAL without limit; the backlog drains over
// however many cycles it takes.
const REAP_BATCH_ROWS = 5000
+// Unindexed, a drained reap rescans the whole heap; retention is days, so minutes of reap lag are harmless.
+const UNINDEXED_REAP_PAUSE_MS = 5 * 60 * 1000
// At ~14 rows/page and ~9 ticks/min, budget reaps ~3M rows/day (~2–3 days for 7.4M backlog); empty walks read ~1MB/tick.
const CONFIRM_RESULT_REAP_BUDGET = {
pagesPerStatement: 16,
@@ -152,6 +154,7 @@ export class RelayCredentialStore {
'committed_at <= ?',
CONFIRM_RESULT_REAP_BUDGET
)
+ private readonly unindexedReapResumesAt = new Map<string, number>()
// Only memory confirms add to it, so its flush timer never starts on a director.
readonly resumeAudit: ResumeConfirmedAuditBuffer
@@ -772,10 +775,11 @@ export class RelayCredentialStore {
// Idempotent reaping stays outside transition transaction or locks last across unrelated batch deletes.
private async reapSettledCredentials(now: number): Promise<void> {
- await this.reapBatch(
+ await this.reapUnindexed(
'relay_invites',
'state IN (?, ?, ?) AND updated_at <= ?',
- ['expired', 'consumed', 'invalidated', now - TERMINAL_INVITE_RETENTION_MS]
+ ['expired', 'consumed', 'invalidated', now - TERMINAL_INVITE_RETENTION_MS],
+ now
)
// Use indexed active/deadline range to avoid scanning 1.5GB drained heap; both readers reject expired deadlines regardless of flag, so one-day-old bases are unusable.
await this.reapBatch('relay_connection_bases', 'active = ? AND deadline <= ?', [
@@ -784,10 +788,11 @@ export class RelayCredentialStore {
])
// consumed_at, not deadline: consumption is what settles this row, and it can happen well
// before the deadline, so measuring from it retains the row for the full window either way.
- await this.reapBatch(
+ await this.reapUnindexed(
'relay_direct_authorizations',
'consumed_at IS NOT NULL AND consumed_at <= ?',
- [now - INACTIVE_AUTHORIZATION_RETENTION_MS]
+ [now - INACTIVE_AUTHORIZATION_RETENTION_MS],
+ now
)
await this.confirmResultReaper.reap(this.database, [now - CONFIRM_RESULT_RETENTION_MS])
// Ordered so the plan walks relay_audit_events_at from its oldest entry: unordered, a LIMIT
@@ -795,23 +800,40 @@ export class RelayCredentialStore {
await this.reapBatch('relay_audit_events', 'at <= ?', [now - AUDIT_EVENT_RETENTION_MS], 'at')
}
+ // Paused before the delete, so an overlapping or failed reap waits too; only a full batch (a backlog) clears it.
+ private async reapUnindexed(
+ table: string,
+ predicate: string,
+ params: unknown[],
+ now: number
+ ): Promise<void> {
+ if (now < (this.unindexedReapResumesAt.get(table) ?? 0)) {
+ return
+ }
+ this.unindexedReapResumesAt.set(table, now + UNINDEXED_REAP_PAUSE_MS)
+ if ((await this.reapBatch(table, predicate, params)) >= REAP_BATCH_ROWS) {
+ this.unindexedReapResumesAt.delete(table)
+ }
+ }
+
// Delete located physical ctid/rowid; PostgreSQL ANY(TID array) guarantees TID scan, while IN can hash-join a full heap scan.
private async reapBatch(
table: string,
predicate: string,
params: unknown[],
orderBy?: string
- ): Promise<void> {
+ ): Promise<number> {
const sqlite = this.database.dialect === 'sqlite'
const address = sqlite ? 'rowid' : 'ctid'
const order = orderBy ? `ORDER BY ${orderBy} ` : ''
const batch = `SELECT ${address} FROM ${table} WHERE ${predicate} ${order}LIMIT ${REAP_BATCH_ROWS}`
- await this.database.query(
+ const rows = await this.database.query(
sqlite
? `DELETE FROM ${table} WHERE rowid IN (${batch})`
: `DELETE FROM ${table} WHERE ctid = ANY(ARRAY(${batch}))`,
params
)
+ return changes(rows)
}
private async installStatusWith(
diff --git a/cloud/apps/relay/src/heap-window-reaper.ts b/cloud/apps/relay/src/heap-window-reaper.ts
index 16829dc3f4d..1c78fdc9860 100644
--- a/cloud/apps/relay/src/heap-window-reaper.ts
+++ b/cloud/apps/relay/src/heap-window-reaper.ts
@@ -80,6 +80,6 @@ async function heapPages(database: RelayDatabase, table: string): Promise<number
return Number(row?.pages ?? 0)
}
-function changes(rows: SqlRow[]): number {
+export function changes(rows: SqlRow[]): number {
return Number(rows[0]?.changes ?? 0)
}