Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure
TypeScript
import { createHash } from 'crypto'
import { BoundedOffPathReporter, type BoundedReporterDropReason } from './boundedOffPathReporter'
import { config } from './config'
import {
infrastructureTelemetryDroppedTotal,
infrastructureTelemetryEnqueuedTotal,
infrastructureTelemetryFlushesTotal,
infrastructureTelemetryGapsTotal,
infrastructureTelemetryRetriesTotal,
} from './metrics'
import { throwForFailedSubmit } from './reporterHttpFailure'
import { signInternalControlJwt } from './utils/internalControlSigner'
export type HccHealthTransitionProjection = {
sourceEventId: string
occurredAt: string
hostLookupReference: HostLookupReference
payload: { transition: string; state: string }
}
/**
* `uid` pins the event to one Host object. A Host deleted and recreated with
* the same name restarts at generation 1; without the uid its events would
* reuse the previous object's identities (#691).
*
* It is required rather than optional because control-api answers a reference
* without one with a 400 this reporter classifies as terminal: the event would
* be dropped, and re-sent and re-dropped on every reconcile pass, since this
* reporter keeps no `seen` set to go quiet. A caller holding a Host snapshot
* that never captured a uid has to decide what to do with it, so the type
* makes the compiler ask instead of letting the reference ship without it.
*/
export type HostLookupReference = {
name: string
namespace: string
generation?: number
uid: string
}
export type HccInfrastructureTelemetryType =
| 'health_transition'
| 'lifecycle_transition'
| 'reconcile_outcome'
| 'controller_error'
export type HccInfrastructureTelemetryPayload = {
reason_code?: string
error_class?: string
phase?: string
state?: string
status?: string
transition?: string
resource_class?: string
summary?: string
detail_ref?: string
attempt?: number
count?: number
}
type HccInfrastructureTelemetryProjectionBase = {
occurredAt: string
hostLookupReference: HostLookupReference
payload?: HccInfrastructureTelemetryPayload
}
type HccReconcileOutcomeProjection = HccInfrastructureTelemetryProjectionBase & {
telemetryType: 'reconcile_outcome'
sourceEventId?: never
}
type HccOccurrenceProjection = HccInfrastructureTelemetryProjectionBase & {
telemetryType: Exclude<HccInfrastructureTelemetryType, 'reconcile_outcome'>
sourceEventId: string
}
export type HccInfrastructureTelemetryProjection =
| HccReconcileOutcomeProjection
| HccOccurrenceProjection
export function hccReconcileOutcomeSourceId(projection: HccReconcileOutcomeProjection): string {
const payload = projection.payload ?? {}
const canonical = [
projection.hostLookupReference.namespace,
projection.hostLookupReference.name,
projection.hostLookupReference.generation ?? 0,
projection.hostLookupReference.uid,
payload.reason_code ?? null,
payload.error_class ?? null,
payload.phase ?? null,
payload.state ?? null,
payload.status ?? null,
payload.transition ?? null,
payload.resource_class ?? null,
payload.summary ?? null,
payload.detail_ref ?? null,
payload.attempt ?? null,
payload.count ?? null,
]
// v3: the tuple gained the Host uid (#691) and lost the gfs_* fields, which
// control-api never accepted (#328). v2 identities are not produced again.
return `hcc-reconcile-outcome-v3:${createHash('sha256').update(JSON.stringify(canonical)).digest('hex')}`
}
export interface InfrastructureTelemetryReporter {
enqueueHealthTransition(projection: HccHealthTransitionProjection): void
enqueue(projection: HccInfrastructureTelemetryProjection): void
stop(timeoutMs?: number): Promise<void>
}
type ReporterDependencies = {
baseUrl?: string
signToken?: () => string
fetchFn?: typeof fetch
capacity?: number
retryLimit?: number
timeoutMs?: number
random?: () => number
deriveReconcileSourceEventId?: (projection: HccReconcileOutcomeProjection) => string
}
type BufferedProjection = HccInfrastructureTelemetryProjection & {
resolvedSourceEventId?: string
}
const DEFAULT_CAPACITY = 64
const DEFAULT_RETRY_LIMIT = 2
const DEFAULT_TIMEOUT_MS = 1_000
const DEFAULT_STOP_TIMEOUT_MS = 1_500
/** Off-path reporter: enqueue is synchronous and does no I/O or serialization. */
export class BoundedInfrastructureTelemetryReporter implements InfrastructureTelemetryReporter {
private readonly queue: BoundedOffPathReporter<BufferedProjection>
private readonly timeoutMs: number
private readonly baseUrl: string
private readonly signToken: () => string
private readonly fetchFn: typeof fetch
private readonly deriveReconcileSourceEventId: (
projection: HccReconcileOutcomeProjection
) => string
constructor(deps: ReporterDependencies = {}) {
const capacity = deps.capacity ?? DEFAULT_CAPACITY
if (!Number.isSafeInteger(capacity) || capacity < 1) {
throw new Error('infrastructure telemetry reporter capacity must be a positive integer')
}
this.timeoutMs = deps.timeoutMs ?? DEFAULT_TIMEOUT_MS
this.baseUrl = (deps.baseUrl ?? config.controlApiBaseUrl).replace(/\/$/, '')
this.signToken = deps.signToken ?? signInternalControlJwt
this.fetchFn = deps.fetchFn ?? fetch
this.deriveReconcileSourceEventId =
deps.deriveReconcileSourceEventId ?? hccReconcileOutcomeSourceId
this.queue = new BoundedOffPathReporter({
capacity,
retryLimit: deps.retryLimit ?? DEFAULT_RETRY_LIMIT,
stopTimeoutMs: DEFAULT_STOP_TIMEOUT_MS,
random: deps.random ?? Math.random,
submit: projection => this.submit(projection),
onEnqueued: projection =>
infrastructureTelemetryEnqueuedTotal.inc({ telemetry_type: projection.telemetryType }),
onAccepted: () => infrastructureTelemetryFlushesTotal.inc({ result: 'accepted' }),
onTerminal: (projection, result) => {
infrastructureTelemetryFlushesTotal.inc({ result })
// Neither terminal result stores this observation: `rejected` is
// refused outright, and `conflict` means a row under that key already
// holds a different payload hash, so ours is dropped. Both are gaps in
// the evidence; the `reason` label keeps them apart (#696).
infrastructureTelemetryGapsTotal.inc({
telemetry_type: projection.telemetryType,
reason: result,
})
},
onRetry: projection =>
infrastructureTelemetryRetriesTotal.inc({ telemetry_type: projection.telemetryType }),
onDrop: (projection, reason) => this.recordDrop(projection.telemetryType, reason),
})
}
enqueueHealthTransition(projection: HccHealthTransitionProjection): void {
this.enqueue({ ...projection, telemetryType: 'health_transition' })
}
enqueue(projection: HccInfrastructureTelemetryProjection): void {
this.queue.enqueue({
...projection,
hostLookupReference: { ...projection.hostLookupReference },
...(projection.payload ? { payload: { ...projection.payload } } : {}),
})
}
private recordDrop(
telemetryType: HccInfrastructureTelemetryType,
reason: BoundedReporterDropReason
): void {
if (reason === 'retry_exhausted' || reason === 'shutdown_submit_failed') {
infrastructureTelemetryFlushesTotal.inc({ result: 'exhausted' })
}
infrastructureTelemetryDroppedTotal.inc({ telemetry_type: telemetryType, reason })
infrastructureTelemetryGapsTotal.inc({ telemetry_type: telemetryType, reason })
}
stop(timeoutMs?: number): Promise<void> {
return this.queue.stop(timeoutMs)
}
private async submit(projection: BufferedProjection): Promise<void> {
if (projection.telemetryType === 'reconcile_outcome' && !projection.resolvedSourceEventId) {
projection.resolvedSourceEventId = this.deriveReconcileSourceEventId(projection)
}
const sourceEventId =
projection.telemetryType === 'reconcile_outcome'
? projection.resolvedSourceEventId
: projection.sourceEventId
if (!sourceEventId) throw new Error('reconcile source identity was not resolved')
const controller = new AbortController()
const timeout = setTimeout(() => controller.abort(), this.timeoutMs)
try {
const response = await this.fetchFn(
`${this.baseUrl}/api/v1/internal/tracing/infrastructure-telemetry-events`,
{
method: 'POST',
headers: {
Authorization: `Bearer ${this.signToken()}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
events: [
{
sourceEventId,
occurredAt: projection.occurredAt,
telemetryType: projection.telemetryType,
hostLookupReference: projection.hostLookupReference,
...(projection.payload ? { payload: projection.payload } : {}),
},
],
}),
signal: controller.signal,
}
)
await throwForFailedSubmit(response, 'telemetry')
} finally {
clearTimeout(timeout)
}
}
}
export function createInfrastructureTelemetryReporter(
enabled: boolean,
deps: ReporterDependencies
): InfrastructureTelemetryReporter | undefined {
if (!enabled) return undefined
return new BoundedInfrastructureTelemetryReporter(deps)
}