Orca is the place for working with a fleet of agents (e.g. Grok Bot, Claude Code). Run any coding agent with your own subscription. Available on desktop, mobile and remote runtime.
TypeScript adeagent-ideai-agentsautomationbotclaude-codecodexghosttygrok-bothermes-agentmobile-appmuseopenclawopencodeorchestrationpiruntimeterminalworktreesyc-backed
import { afterEach, describe, expect, it, vi } from 'vitest'
import type { RelayDatabase } from './database.js'
import { observeRelayDatabase } from './observed-relay-database.js'
import {
CONTROL_RTT_RESERVOIR_LIMIT,
observedRelayRequests,
percentile,
PRE_AUTH_RESERVOIR_LIMIT,
RelayObservability,
STICKY_CONNECTION_LIMIT,
type RelayProcessCounts
} from './relay-observability.js'
import { RELAY_FIX_LEVEL } from './relay-fix-level.js'
const counts: RelayProcessCounts = {
totalConnections: 9,
preAuthConnections: 1,
controls: 2,
splices: 3,
pendingSplices: 1,
queuedBytes: 4096,
databasePoolTotal: 3,
databasePoolIdle: 0,
databasePoolWaiting: 2,
databasePoolWaitersMax: 3,
databasePoolOldestWaitMs: 750,
databasePoolWaitMsMax: 1_250
}
// Alias only legitimate credential schema keys in a clone so raw secret fields/values elsewhere still trip the guard.
const SCHEMA_KEY_ALIASES: Record<string, string> = {
clientAcceptCredentialMsP95: 'clientAcceptStageTwoMsP95'
}
function scrubSchemaKeys(entries: Array<Record<string, unknown>>): string {
return JSON.stringify(
entries.map((entry) =>
Object.fromEntries(
Object.entries(entry).map(([key, value]) => [
SCHEMA_KEY_ALIASES[key] ?? key,
key === 'clientAcceptsAbandonedByStageDelta' ? renameStageKeys(value) : value
])
)
)
)
}
function renameStageKeys(bucket: unknown): unknown {
if (bucket === null || typeof bucket !== 'object') return bucket
return Object.fromEntries(
Object.entries(bucket).map(([stage, count]) => [
stage === 'credential' ? 'stageTwo' : stage,
count
])
)
}
describe('relay observability', () => {
afterEach(() => vi.restoreAllMocks())
it('stamps every runtime metrics line with the fix level', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c25', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.flush(counts)
expect(entries[0]).toMatchObject({
event: 'orca_relay_runtime_metrics',
fixLevel: RELAY_FIX_LEVEL
})
})
// Why: a director whose loop froze for 134 s still reported a 68 ms p99 (staging 2026-10-11).
it('reports one long event-loop freeze as the window max', async () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'director', cellId: 'director', region: 'us-central1' },
(entry) => entries.push(entry)
)
observability.start(() => counts, 3_600_000)
await new Promise((resolve) => setTimeout(resolve, 100))
const frozenUntil = performance.now() + 300
while (performance.now() < frozenUntil) {
// Holds the loop the way a blocking connect() does.
}
await new Promise((resolve) => setTimeout(resolve, 100))
observability.flush(counts)
observability.stop()
expect(entries[0]!.eventLoopDelayMsMax).toBeGreaterThanOrEqual(250)
})
it('emits safe readiness dependency outcomes', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.recordReadiness({
ready: false,
failure: 'sql_failed',
jwksLatencyMs: 12,
sqlLatencyMs: 2_001,
totalLatencyMs: 2_013
})
expect(entries).toEqual([
{
severity: 'WARNING',
message: 'Orca Relay readiness check',
event: 'orca_relay_readiness_check',
metricVersion: 1,
role: 'cell',
cellId: 'production-gce-c28',
region: 'asia-east2',
ready: false,
failure: 'sql_failed',
jwksLatencyMs: 12,
sqlLatencyMs: 2_001,
totalLatencyMs: 2_013
}
])
})
it('flags a readiness answer served from the last known good probe', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.recordReadiness({
ready: true,
degraded: true,
degradedDependencies: ['jwks'],
failure: 'jwks_timed_out',
jwksLatencyMs: 2_001,
sqlLatencyMs: 4,
totalLatencyMs: 2_002
})
expect(entries).toEqual([
expect.objectContaining({
severity: 'WARNING',
event: 'orca_relay_readiness_check',
ready: true,
degraded: true,
degradedDependencies: ['jwks'],
failure: 'jwks_timed_out'
})
])
})
it('separates entering the readiness grace window from leaving it', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.recordReadinessGrace({
dependency: 'sql',
grace: 'entered',
failure: 'sql_failed',
lastSuccessAgeMs: 12_000,
graceMs: 180_000
})
observability.recordReadinessGrace({
dependency: 'sql',
grace: 'recovered',
lastSuccessAgeMs: 0,
graceMs: 180_000
})
expect(entries).toEqual([
{
severity: 'WARNING',
message: 'Orca Relay readiness entered last-known-good grace',
event: 'orca_relay_readiness_grace_entered',
metricVersion: 1,
role: 'cell',
cellId: 'production-gce-c28',
region: 'asia-east2',
dependency: 'sql',
grace: 'entered',
failure: 'sql_failed',
lastSuccessAgeMs: 12_000,
graceMs: 180_000
},
{
severity: 'INFO',
message: 'Orca Relay readiness left last-known-good grace',
event: 'orca_relay_readiness_grace_left',
metricVersion: 1,
role: 'cell',
cellId: 'production-gce-c28',
region: 'asia-east2',
dependency: 'sql',
grace: 'recovered',
lastSuccessAgeMs: 0,
graceMs: 180_000
}
])
})
it('excludes sockets stuck in closing state from observed relay work', () => {
expect(observedRelayRequests(counts)).toBe(7)
})
it('keeps rejection reasons separate per lane and resets them each flush', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'director', cellId: 'director', region: 'us-central1' },
(entry) => entries.push(entry)
)
observability.recordAssignmentAdmission('placement-rejected')
observability.recordAssignmentRejectionReason('placement', 'host-rate-limited')
observability.recordAssignmentRejectionReason('placement', 'host-rate-limited')
observability.recordAssignmentRejectionReason('placement', 'queue-full')
observability.recordAssignmentRejectionReason('sticky', 'wait-timeout')
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
placementAssignmentRejectionsDelta: 1,
placementRejectionsByReasonDelta: { 'host-rate-limited': 2, 'queue-full': 1 },
stickyRejectionsByReasonDelta: { 'wait-timeout': 1 }
})
expect(entries[1]).toMatchObject({
placementRejectionsByReasonDelta: {},
stickyRejectionsByReasonDelta: {}
})
})
it('splits non-drain 503s from scheduled drain-return deferrals', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'director', cellId: 'director', region: 'us-central1' },
(entry) => entries.push(entry)
)
for (let index = 0; index < 40; index++) {
observability.recordAssignmentUnavailable('drain-return-deferred')
}
observability.recordAssignmentUnavailable('reserve-paced-drained')
observability.recordAssignmentUnavailable('sticky-lane')
observability.recordAssignmentUnavailable('relay_capacity_exhausted')
observability.recordAssignmentUnavailable('relay_capacity_exhausted')
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
assign503sByCauseDelta: {
'drain-return-deferred': 40,
'reserve-paced-drained': 1,
'sticky-lane': 1,
relay_capacity_exhausted: 2
},
assignNonDrain503sDelta: 3
})
expect(entries[1]).toMatchObject({ assign503sByCauseDelta: {}, assignNonDrain503sDelta: 0 })
})
it('reports lane slot service times only for windows that served the lane', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'director', cellId: 'director', region: 'us-central1' },
(entry) => entries.push(entry)
)
for (let ms = 1; ms <= 100; ms++) {
observability.recordAdmissionServiceMs('sticky', ms)
observability.recordAdmissionServiceMs('drain-return', ms * 10)
}
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
stickyServiceMsP50: 50,
stickyServiceMsP99: 99,
drainReturnServiceMsP50: 500,
drainReturnServiceMsP95: 950
})
expect(entries[1]).not.toHaveProperty('stickyServiceMsP50')
expect(entries[1]).not.toHaveProperty('drainReturnServiceMsP50')
})
it('summarises sticky slots per window and per pooled connection', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'director', cellId: 'director', region: 'us-central1' },
(entry) => entries.push(entry)
)
for (let ms = 1; ms <= 100; ms++) {
observability.recordStickySlot({
queueWaitMs: ms * 20,
poolWaitMs: ms,
sqlMs: ms * 2,
statements: 14,
// The slow connection serves every other request.
statementMsByConnection: new Map([ms % 2 === 0 ? [1, [300]] : [2, [5, 6]]])
})
}
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
stickySlotsDelta: 100,
stickyQueueWaitMsP50: 1_000,
stickyQueueWaitMsP99: 1_980,
stickyQueueWaitMsMax: 2_000,
stickyPoolWaitMsP50: 50,
stickyPoolWaitMsP99: 99,
stickySqlMsP50: 100,
stickySqlMsP99: 198,
stickyStatementsP50: 14,
stickyStatementsMax: 14,
// 50 statements at 300 ms and 100 at 5-6 ms: the pooled median is the fast connection's.
stickyStatementMsP50: 6,
stickySqlByConnection: {
'1': {
requests: 50,
statements: 50,
statementMsP50: 300,
statementMsP99: 300,
statementMsMax: 300
},
'2': { requests: 50, statements: 100, statementMsP50: 5, statementMsP99: 6 }
},
stickySqlConnectionSamplesDroppedDelta: 0
})
expect(entries[1]).not.toHaveProperty('stickySlotsDelta')
expect(entries[1]).not.toHaveProperty('stickySqlByConnection')
})
it('stamps the instance zone once it is known', () => {
const entries: Record<string, unknown>[] = []
const observability = new RelayObservability(
{ role: 'director', cellId: 'director', region: 'us-central1' },
(entry) => entries.push(entry)
)
observability.flush(counts)
observability.setInstanceZone('us-central1-a')
observability.flush(counts)
expect(entries[0]).not.toHaveProperty('instanceZone')
expect(entries[1]).toMatchObject({ instanceZone: 'us-central1-a' })
})
it('adds the sandbox TCP counter deltas of each window once a reader is set', () => {
const entries: Record<string, unknown>[] = []
const observability = new RelayObservability(
{ role: 'director', cellId: 'director', region: 'us-central1' },
(entry) => entries.push(entry)
)
const reads = [10, 13, 13, undefined, 20, 2].map((retransSegs) =>
retransSegs === undefined
? undefined
: { inSegs: retransSegs * 100, outSegs: 0, retransSegs, inErrs: 0, outRsts: 0, estabResets: 0 }
)
observability.flush(counts)
observability.setTcpCounterReader(() => reads.shift())
for (let window = 0; window < 5; window++) {
observability.flush(counts)
}
expect(entries[0]).not.toHaveProperty('tcpRetransSegsDelta')
expect(entries[1]).toMatchObject({ tcpRetransSegsDelta: 3, tcpInSegsDelta: 300 })
expect(entries[2]).toMatchObject({ tcpRetransSegsDelta: 0 })
// An unreadable window, the window after it, and a counter reset carry no delta.
expect(entries[3]).not.toHaveProperty('tcpRetransSegsDelta')
expect(entries[4]).not.toHaveProperty('tcpRetransSegsDelta')
expect(entries[5]).not.toHaveProperty('tcpRetransSegsDelta')
})
it('bounds the per-connection sticky summary', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'director', cellId: 'director', region: 'us-central1' },
(entry) => entries.push(entry)
)
for (let id = 1; id <= STICKY_CONNECTION_LIMIT + 3; id++) {
observability.recordStickySlot({
queueWaitMs: 0,
poolWaitMs: 0,
sqlMs: 1,
statements: 1,
statementMsByConnection: new Map([[id, [1]]])
})
}
observability.flush(counts)
expect(Object.keys(entries[0]?.stickySqlByConnection as object)).toHaveLength(
STICKY_CONNECTION_LIMIT
)
expect(entries[0]).toMatchObject({ stickySqlConnectionSamplesDroppedDelta: 3 })
})
it('sums lock waiters per role and table over the samples taken', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'director', cellId: 'director', region: 'us-central1' },
(entry) => entries.push(entry)
)
observability.recordDatabaseLockWaitSample([
{ waiterRole: 'cell', table: 'relay_cells', holderRole: 'cell', waiters: 3 },
{ waiterRole: 'director', table: 'relay_cells', holderRole: 'cell', waiters: 1 },
{ waiterRole: 'cell', table: 'relay_assignments', holderRole: 'director', waiters: 2 }
])
observability.recordDatabaseLockWaitSample([])
observability.recordDatabaseLockWaitSample([
{ waiterRole: 'cell', table: 'relay_cells', holderRole: 'cell', waiters: 1 }
])
observability.flush(counts)
expect(entries[0]).toMatchObject({
dbLockWaitSamplesDelta: 3,
dbLockWaitersByKeyDelta: {
'cell:relay_cells:cell': 4,
'director:relay_cells:cell': 1,
'cell:relay_assignments:director': 2
},
dbCellRowLockWaitersDirectorDelta: 1,
dbCellRowLockWaitersCellDelta: 4
})
})
it('aggregates coarse region requests, selections, fallbacks, and outages', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'director', cellId: 'director', region: 'us-central1' },
(entry) => entries.push(entry)
)
observability.recordRegionRequest('asia-east2')
observability.recordRegionRequest(undefined)
observability.recordRegionSelection({
targetRegion: 'asia-east2',
selectedRegion: 'us-central1',
fallback: true
})
observability.recordRegionSelection({ targetRegion: 'asia-east2', fallback: false })
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
requestedRegionsDelta: { 'asia-east2': 1, unhinted: 1 },
selectedRegionsDelta: { 'us-central1': 1 },
regionFallbacksDelta: { 'asia-east2': 1 },
unavailableRegionsDelta: { 'asia-east2': 1 },
// Flat per-region siblings the log-based metrics extract; `unhinted` stays map-only.
requestedRegionUsCentral1Delta: 0,
requestedRegionAsiaEast2Delta: 1,
selectedRegionUsCentral1Delta: 1,
selectedRegionAsiaEast2Delta: 0
})
expect(entries[1]).toMatchObject({
requestedRegionsDelta: {},
selectedRegionsDelta: {},
regionFallbacksDelta: {},
unavailableRegionsDelta: {},
// Zeros keep publishing so an idle window cannot drop a series out of the skew join.
requestedRegionUsCentral1Delta: 0,
requestedRegionAsiaEast2Delta: 0,
selectedRegionUsCentral1Delta: 0,
selectedRegionAsiaEast2Delta: 0
})
})
it('emits bounded aggregate runtime signals without identities or credentials', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'staging-c1', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.recordAuth(true)
observability.recordAuth(false)
observability.recordForwardedBytes(123)
observability.recordHttp(45.6789)
observability.recordReconnect()
observability.recordSql(12.3456, true)
observability.recordSql(4, false)
observability.recordControlRenewal(2, 'renewed')
observability.recordControlRenewal(8, 'control_activity_not_found')
observability.recordControlRenewal(4, 'renewed')
observability.recordControlActivityRecovery(true)
observability.recordControlActivityRecovery(false)
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
event: 'orca_relay_runtime_metrics',
metricVersion: 2,
role: 'cell',
cellId: 'staging-c1',
region: 'asia-east2',
...counts,
forwardedBytesDelta: 123,
authSuccessesDelta: 1,
authFailuresDelta: 1,
reconnectsDelta: 1,
sqlQueriesDelta: 2,
sqlFailuresDelta: 1,
sqlLatencyMsMax: 12.346,
controlRenewalsByOutcomeDelta: { renewed: 2, control_activity_not_found: 1 },
controlRenewalsDelta: 3,
controlRenewalSuccessesDelta: 2,
controlRenewalLeaseMissesDelta: 1,
controlRenewalLatencyMsP50: 4,
controlRenewalLatencyMsP95: 8,
controlRenewalLatencyMsMax: 8,
controlActivityRecoveriesDelta: 1,
controlActivityRecoveryFailuresDelta: 1,
httpLatencyMsMax: 45.679
})
expect(entries[1]).toMatchObject({
forwardedBytesDelta: 0,
authSuccessesDelta: 0,
authFailuresDelta: 0,
reconnectsDelta: 0,
sqlQueriesDelta: 0,
sqlFailuresDelta: 0,
sqlLatencyMsMax: 0,
controlRenewalsByOutcomeDelta: {},
controlRenewalsDelta: 0,
controlRenewalSuccessesDelta: 0,
controlRenewalLeaseMissesDelta: 0,
controlRenewalLatencyMsP50: 0,
controlRenewalLatencyMsP95: 0,
controlRenewalLatencyMsMax: 0,
controlActivityRecoveriesDelta: 0,
controlActivityRecoveryFailuresDelta: 0,
httpLatencyMsMax: 0
})
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
})
it('aggregates control and splice closes as bounded per-reason deltas', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'staging-c1', region: 'us-central1' },
(entry) => entries.push(entry)
)
observability.recordControlClose(1006)
observability.recordControlClose(1006)
observability.recordControlClose(4402)
observability.recordPostProofClose('relay draining')
observability.recordPostProofClose('relay draining')
observability.recordSpliceClose('host-oversize-frame')
observability.recordSpliceClose('queue-limit')
observability.recordClientAcceptAbandoned('activity', 14_250.4)
observability.recordClientAcceptAbandoned('activity', 2_000)
observability.recordClientAcceptAbandoned('credential', 3_000)
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
controlClosesByCodeDelta: { 1006: 2, 4402: 1 },
postProofClosesByReasonDelta: { 'relay draining': 2 },
spliceClosesByTriggerDelta: { 'host-oversize-frame': 1, 'queue-limit': 1 },
clientAcceptsAbandonedByStageDelta: { activity: 2, credential: 1 },
clientAcceptAbandonedMsMax: 14_250.4
})
expect(entries[1]).toMatchObject({
controlClosesByCodeDelta: {},
postProofClosesByReasonDelta: {},
spliceClosesByTriggerDelta: {},
clientAcceptsAbandonedByStageDelta: {},
clientAcceptAbandonedMsMax: 0
})
})
it('counts the phone path, phone refusals by code and the step-6 failures per window', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
const stageMs = { assignment: 0, credential: 170, activity: 0, attach: 300, basis: 0 }
observability.recordClientAcceptCompleted({ totalMs: 470, stageMs, phonePath: 'memory' })
observability.recordClientAcceptCompleted({ totalMs: 3_600, stageMs, phonePath: 'db' })
observability.recordPhoneRejection(4401)
observability.recordPhoneRejection(4404)
observability.recordPhoneRejection(4404)
observability.recordPhoneAttachDeadline()
observability.recordPhoneAcceptDatabaseError()
observability.recordResumeConfirmFailure('reject-revoked')
observability.recordResumeRenewalFailure()
observability.flush({ ...counts, resumeAuditDropped: 3 })
observability.flush(counts)
expect(entries[0]).toMatchObject({
clientAcceptsByPhonePathDelta: { memory: 1, db: 1 },
phoneRejectionsByCodeDelta: { '4401': 1, '4404': 2 },
phoneAttachDeadlinesDelta: 1,
phoneAcceptDatabaseErrorsDelta: 1,
resumeConfirmFailuresByCodeDelta: { 'reject-revoked': 1 },
resumeRenewalFailuresDelta: 1,
resumeAuditDropped: 3
})
expect(entries[1]).toMatchObject({
phoneRejectionsByCodeDelta: {},
phoneAttachDeadlinesDelta: 0,
phoneAcceptDatabaseErrorsDelta: 0,
resumeConfirmFailuresByCodeDelta: {}
})
})
it('summarises completed client accepts and control round trips per window', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
observability.recordClientAcceptCompleted({
totalMs: 812.4567,
stageMs: { assignment: 120, credential: 90, activity: 40, attach: 500, basis: 62 }
})
observability.recordClientAcceptCompleted({
totalMs: 6_400,
stageMs: { assignment: 4_100, credential: 95, activity: 60, attach: 2_000, basis: 145 }
})
observability.recordControlRtt(28)
observability.recordControlRtt(240)
observability.recordControlRtt(31)
observability.flush(counts)
observability.flush(counts)
expect(entries[0]).toMatchObject({
clientAcceptCompletedDelta: 2,
clientAcceptTotalMsP50: 812.457,
clientAcceptTotalMsP95: 6_400,
clientAcceptTotalMsMax: 6_400,
clientAcceptAssignmentMsP95: 4_100,
clientAcceptCredentialMsP95: 95,
clientAcceptActivityMsP95: 60,
clientAcceptAttachMsP95: 2_000,
clientAcceptBasisMsP95: 145,
controlRttSamplesDelta: 3,
controlRttMsP50: 31,
controlRttMsP95: 240,
controlRttMsMax: 240
})
expect(entries[0]).toMatchObject({
event: 'orca_relay_runtime_metrics',
metricVersion: 2,
clientAcceptsAbandonedByStageDelta: {},
clientAcceptAbandonedMsMax: 0
})
expect(entries[0]).toMatchObject({ clientAcceptsByPhonePathDelta: {} })
// Omit empty percentiles: aggregated zeros are indistinguishable from actual zero-latency samples.
expect(entries[1]).toMatchObject({ clientAcceptCompletedDelta: 0, controlRttSamplesDelta: 0 })
for (const omitted of [
'clientAcceptTotalMsP50',
'clientAcceptTotalMsP95',
'clientAcceptTotalMsMax',
'clientAcceptAssignmentMsP95',
'clientAcceptCredentialMsP95',
'clientAcceptActivityMsP95',
'clientAcceptAttachMsP95',
'clientAcceptBasisMsP95',
'controlRttMsP50',
'controlRttMsP95',
'controlRttMsMax'
]) {
expect(entries[1]).not.toHaveProperty(omitted)
expect(entries[0]).toHaveProperty(omitted)
}
expect(scrubSchemaKeys(entries)).not.toMatch(/token|credential|userId|relayHostId/i)
})
it('caps the control round-trip reservoir and reports what it dropped', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
const flooded = CONTROL_RTT_RESERVOIR_LIMIT * 20
for (let sample = 0; sample < flooded; sample++) {
observability.recordControlRtt(10 + (sample % 40))
}
observability.flush(counts)
expect(entries[0]).toMatchObject({
controlRttSamplesDelta: flooded,
controlRttSamplesDroppedDelta: flooded - CONTROL_RTT_RESERVOIR_LIMIT
})
expect(entries[0]!.controlRttMsP50 as number).toBeGreaterThanOrEqual(10)
expect(entries[0]!.controlRttMsMax as number).toBeLessThanOrEqual(49)
observability.flush(counts)
expect(entries[1]).toMatchObject({
controlRttSamplesDelta: 0,
controlRttSamplesDroppedDelta: 0
})
expect(entries[1]).not.toHaveProperty('controlRttMsP50')
})
it('samples the whole flooded window rather than its first samples', () => {
let seed = 42
vi.spyOn(Math, 'random').mockImplementation(() => {
seed = (Math.imul(seed, 1_664_525) + 1_013_904_223) >>> 0
return seed / 0x1_0000_0000
})
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'production-gce-c28', region: 'asia-east2' },
(entry) => entries.push(entry)
)
const half = CONTROL_RTT_RESERVOIR_LIMIT * 10
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(10)
for (let sample = 0; sample < half; sample++) observability.recordControlRtt(900)
observability.flush(counts)
// A first-N reservoir would publish only the early 10 ms samples.
expect(entries[0]!.controlRttMsP95).toBe(900)
expect(entries[0]!.controlRttMsMax).toBe(900)
})
it('summarises host pre-auth latency per path and outcome', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'staging-gce-c1', region: 'us-central1' },
(entry) => entries.push(entry)
)
for (let ms = 1; ms <= 100; ms++) observability.recordHostPreAuth('db', 'admitted', ms)
observability.recordHostPreAuth('db', 'abandoned', 6_000)
observability.recordHostPreAuth('beforeCheck', 'refused', 2_000)
observability.recordHostPreAuth('reserve', 'admitted', 12.3456)
observability.flush(counts)
observability.flush(counts)
const none = { admitted: { count: 0 }, refused: { count: 0 }, abandoned: { count: 0 } }
const buckets = (filled: Partial<Record<string, number>>): Record<string, number> => ({
'50': 0,
'100': 0,
'200': 0,
'300': 0,
'450': 0,
'700': 0,
'1000': 0,
'2000': 0,
'5000': 0,
'+Inf': 0,
...filled
})
expect(entries[0]!.preAuthLatencyMs).toEqual({
beforeCheck: {
...none,
refused: {
count: 1,
p50: 2_000,
p95: 2_000,
p99: 2_000,
max: 2_000,
bucketsLeMs: buckets({ '2000': 1 })
}
},
db: {
...none,
admitted: {
count: 100,
p50: 50,
p95: 95,
p99: 99,
max: 100,
bucketsLeMs: buckets({ '50': 50, '100': 50 })
},
abandoned: {
count: 1,
p50: 6_000,
p95: 6_000,
p99: 6_000,
max: 6_000,
bucketsLeMs: buckets({ '+Inf': 1 })
}
},
reserve: {
...none,
admitted: {
count: 1,
p50: 12.346,
p95: 12.346,
p99: 12.346,
max: 12.346,
bucketsLeMs: buckets({ '50': 1 })
}
}
})
expect(entries[1]!.preAuthLatencyMs).toEqual({ beforeCheck: none, db: none, reserve: none })
})
it('counts every hello in the pre-auth buckets, inclusive of each bound', () => {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'staging-gce-c1', region: 'us-central1' },
(entry) => entries.push(entry)
)
const flooded = PRE_AUTH_RESERVOIR_LIMIT * 5
for (let sample = 0; sample < flooded; sample++) {
observability.recordHostPreAuth('reserve', 'admitted', 450)
}
observability.recordHostPreAuth('reserve', 'admitted', 450.001)
observability.recordHostPreAuth('reserve', 'admitted', 9_000)
observability.flush(counts)
expect(entries[0]).toMatchObject({
preAuthLatencyMs: {
reserve: {
admitted: {
// Exact past the reservoir cap, even when it declined the slowest hello.
count: flooded + 2,
max: 9_000,
p50: 450,
bucketsLeMs: { '300': 0, '450': flooded, '700': 1, '5000': 0, '+Inf': 1 }
}
}
}
})
})
it('observes successful and failed database calls including transactions', async () => {
const recordSql = vi.fn()
const underlying: RelayDatabase = {
query: vi.fn(async () => [{ ok: true }]),
queryLocked: vi.fn(async (sql, _params, options) => {
throw new Error(
options?.failIfUnavailable && sql === 'SELECT 3'
? 'database_lock_unavailable'
: 'database unavailable'
)
}),
transaction: async (operation) => await operation(underlying),
close: vi.fn(async () => {})
}
const database = observeRelayDatabase(underlying, {
recordAuth: vi.fn(),
recordForwardedBytes: vi.fn(),
recordHttp: vi.fn(),
recordReconnect: vi.fn(),
recordSql
})
await database.query('SELECT 1')
await expect(
database.transaction(async (tx) => await tx.queryLocked('SELECT 2'))
).rejects.toThrow('database unavailable')
await expect(database.queryLocked('SELECT 3', [], { failIfUnavailable: true })).rejects.toThrow(
'database_lock_unavailable'
)
await expect(database.queryLocked('SELECT 4', [], { failIfUnavailable: true })).rejects.toThrow(
'database unavailable'
)
expect(recordSql).toHaveBeenCalledTimes(4)
expect(recordSql.mock.calls.map((call) => call[1])).toEqual([true, false, true, false])
})
})
function summaryThroughFlush(samples: number[]) {
const entries: Array<Record<string, unknown>> = []
const observability = new RelayObservability(
{ role: 'cell', cellId: 'staging-c1', region: 'us-central1' },
(entry) => entries.push(entry)
)
for (const sample of samples) observability.recordControlRenewal(sample, 'renewed')
observability.flush(counts)
const entry = entries[0]!
return {
p50: entry.controlRenewalLatencyMsP50,
p95: entry.controlRenewalLatencyMsP95,
max: entry.controlRenewalLatencyMsMax
}
}
describe('latency window summarisation', () => {
it.each([
{ name: 'empty', values: [], expected: { p50: 0, p95: 0, max: 0 } },
{ name: 'single', values: [7.5], expected: { p50: 7.5, p95: 7.5, max: 7.5 } },
{ name: 'unsorted', values: [10, 1, 5, 3], expected: { p50: 3, p95: 10, max: 10 } },
{ name: 'duplicates', values: [4, 4, 4], expected: { p50: 4, p95: 4, max: 4 } },
{ name: 'negative', values: [-5, -1, -9, -3, -2], expected: { p50: -3, p95: -1, max: 0 } },
{
name: 'mixed signs',
values: [-2, 3, -7, 0, 11, -0.5],
expected: { p50: -0.5, p95: 11, max: 11 }
},
{ name: 'signed zero', values: [0, -0, -0, 0], expected: { p50: 0, p95: 0, max: 0 } },
{
name: 'NaN',
values: [Number.NaN, Number.NaN],
expected: { p50: Number.NaN, p95: Number.NaN, max: Number.NaN }
},
{
name: 'positive infinity',
values: [Infinity, 3, 1],
expected: { p50: 3, p95: Infinity, max: Infinity }
},
{
name: 'negative infinity',
values: [-Infinity, -3, -1],
expected: { p50: -3, p95: -1, max: 0 }
},
{
name: 'both infinities',
values: [Infinity, -Infinity, 3, -Infinity],
expected: { p50: -Infinity, p95: Infinity, max: Infinity }
},
{
name: 'rounding',
values: [0.00049, 0.0005, 0.00051, 0.9995],
expected: { p50: 0.001, p95: 1, max: 1 }
}
])('publishes explicit p50, p95 and maximum for $name samples', ({ values, expected }) => {
expect(summaryThroughFlush(values)).toEqual(expected)
})
it('leaves the caller window untouched when computing a percentile', () => {
const samples = [5, -1, Number.NaN, 3, -0]
const before = [...samples]
percentile(samples, 0.5)
expect(samples).toEqual(before)
})
})