Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
9.1 KB
Raw Blame
import { createHash } from 'crypto'
import { BoundedOffPathReporter, type BoundedReporterDropReason } from './boundedOffPathReporter'
import { config } from './config'
import {
  infrastructureTelemetryDroppedTotal,
  infrastructureTelemetryEnqueuedTotal,
  infrastructureTelemetryFlushesTotal,
  infrastructureTelemetryGapsTotal,
  infrastructureTelemetryRetriesTotal,
} from './metrics'
import { throwForFailedSubmit } from './reporterHttpFailure'
import { signInternalControlJwt } from './utils/internalControlSigner'

export type HccHealthTransitionProjection = {
  sourceEventId: string
  occurredAt: string
  hostLookupReference: HostLookupReference
  payload: { transition: string; state: string }
}

/**
 * `uid` pins the event to one Host object. A Host deleted and recreated with
 * the same name restarts at generation 1; without the uid its events would
 * reuse the previous object's identities (#691).
 *
 * It is required rather than optional because control-api answers a reference
 * without one with a 400 this reporter classifies as terminal: the event would
 * be dropped, and re-sent and re-dropped on every reconcile pass, since this
 * reporter keeps no `seen` set to go quiet. A caller holding a Host snapshot
 * that never captured a uid has to decide what to do with it, so the type
 * makes the compiler ask instead of letting the reference ship without it.
 */
export type HostLookupReference = {
  name: string
  namespace: string
  generation?: number
  uid: string
}

export type HccInfrastructureTelemetryType =
  | 'health_transition'
  | 'lifecycle_transition'
  | 'reconcile_outcome'
  | 'controller_error'

export type HccInfrastructureTelemetryPayload = {
  reason_code?: string
  error_class?: string
  phase?: string
  state?: string
  status?: string
  transition?: string
  resource_class?: string
  summary?: string
  detail_ref?: string
  attempt?: number
  count?: number
}

type HccInfrastructureTelemetryProjectionBase = {
  occurredAt: string
  hostLookupReference: HostLookupReference
  payload?: HccInfrastructureTelemetryPayload
}

type HccReconcileOutcomeProjection = HccInfrastructureTelemetryProjectionBase & {
  telemetryType: 'reconcile_outcome'
  sourceEventId?: never
}

type HccOccurrenceProjection = HccInfrastructureTelemetryProjectionBase & {
  telemetryType: Exclude<HccInfrastructureTelemetryType, 'reconcile_outcome'>
  sourceEventId: string
}

export type HccInfrastructureTelemetryProjection =
  | HccReconcileOutcomeProjection
  | HccOccurrenceProjection

export function hccReconcileOutcomeSourceId(projection: HccReconcileOutcomeProjection): string {
  const payload = projection.payload ?? {}
  const canonical = [
    projection.hostLookupReference.namespace,
    projection.hostLookupReference.name,
    projection.hostLookupReference.generation ?? 0,
    projection.hostLookupReference.uid,
    payload.reason_code ?? null,
    payload.error_class ?? null,
    payload.phase ?? null,
    payload.state ?? null,
    payload.status ?? null,
    payload.transition ?? null,
    payload.resource_class ?? null,
    payload.summary ?? null,
    payload.detail_ref ?? null,
    payload.attempt ?? null,
    payload.count ?? null,
  ]
  // v3: the tuple gained the Host uid (#691) and lost the gfs_* fields, which
  // control-api never accepted (#328). v2 identities are not produced again.
  return `hcc-reconcile-outcome-v3:${createHash('sha256').update(JSON.stringify(canonical)).digest('hex')}`
}

export interface InfrastructureTelemetryReporter {
  enqueueHealthTransition(projection: HccHealthTransitionProjection): void
  enqueue(projection: HccInfrastructureTelemetryProjection): void
  stop(timeoutMs?: number): Promise<void>
}

type ReporterDependencies = {
  baseUrl?: string
  signToken?: () => string
  fetchFn?: typeof fetch
  capacity?: number
  retryLimit?: number
  timeoutMs?: number
  random?: () => number
  deriveReconcileSourceEventId?: (projection: HccReconcileOutcomeProjection) => string
}

type BufferedProjection = HccInfrastructureTelemetryProjection & {
  resolvedSourceEventId?: string
}

const DEFAULT_CAPACITY = 64
const DEFAULT_RETRY_LIMIT = 2
const DEFAULT_TIMEOUT_MS = 1_000
const DEFAULT_STOP_TIMEOUT_MS = 1_500

/** Off-path reporter: enqueue is synchronous and does no I/O or serialization. */
export class BoundedInfrastructureTelemetryReporter implements InfrastructureTelemetryReporter {
  private readonly queue: BoundedOffPathReporter<BufferedProjection>
  private readonly timeoutMs: number
  private readonly baseUrl: string
  private readonly signToken: () => string
  private readonly fetchFn: typeof fetch
  private readonly deriveReconcileSourceEventId: (
    projection: HccReconcileOutcomeProjection
  ) => string

  constructor(deps: ReporterDependencies = {}) {
    const capacity = deps.capacity ?? DEFAULT_CAPACITY
    if (!Number.isSafeInteger(capacity) || capacity < 1) {
      throw new Error('infrastructure telemetry reporter capacity must be a positive integer')
    }
    this.timeoutMs = deps.timeoutMs ?? DEFAULT_TIMEOUT_MS
    this.baseUrl = (deps.baseUrl ?? config.controlApiBaseUrl).replace(/\/$/, '')
    this.signToken = deps.signToken ?? signInternalControlJwt
    this.fetchFn = deps.fetchFn ?? fetch
    this.deriveReconcileSourceEventId =
      deps.deriveReconcileSourceEventId ?? hccReconcileOutcomeSourceId
    this.queue = new BoundedOffPathReporter({
      capacity,
      retryLimit: deps.retryLimit ?? DEFAULT_RETRY_LIMIT,
      stopTimeoutMs: DEFAULT_STOP_TIMEOUT_MS,
      random: deps.random ?? Math.random,
      submit: projection => this.submit(projection),
      onEnqueued: projection =>
        infrastructureTelemetryEnqueuedTotal.inc({ telemetry_type: projection.telemetryType }),
      onAccepted: () => infrastructureTelemetryFlushesTotal.inc({ result: 'accepted' }),
      onTerminal: (projection, result) => {
        infrastructureTelemetryFlushesTotal.inc({ result })
        // Neither terminal result stores this observation: `rejected` is
        // refused outright, and `conflict` means a row under that key already
        // holds a different payload hash, so ours is dropped. Both are gaps in
        // the evidence; the `reason` label keeps them apart (#696).
        infrastructureTelemetryGapsTotal.inc({
          telemetry_type: projection.telemetryType,
          reason: result,
        })
      },
      onRetry: projection =>
        infrastructureTelemetryRetriesTotal.inc({ telemetry_type: projection.telemetryType }),
      onDrop: (projection, reason) => this.recordDrop(projection.telemetryType, reason),
    })
  }

  enqueueHealthTransition(projection: HccHealthTransitionProjection): void {
    this.enqueue({ ...projection, telemetryType: 'health_transition' })
  }

  enqueue(projection: HccInfrastructureTelemetryProjection): void {
    this.queue.enqueue({
      ...projection,
      hostLookupReference: { ...projection.hostLookupReference },
      ...(projection.payload ? { payload: { ...projection.payload } } : {}),
    })
  }

  private recordDrop(
    telemetryType: HccInfrastructureTelemetryType,
    reason: BoundedReporterDropReason
  ): void {
    if (reason === 'retry_exhausted' || reason === 'shutdown_submit_failed') {
      infrastructureTelemetryFlushesTotal.inc({ result: 'exhausted' })
    }
    infrastructureTelemetryDroppedTotal.inc({ telemetry_type: telemetryType, reason })
    infrastructureTelemetryGapsTotal.inc({ telemetry_type: telemetryType, reason })
  }

  stop(timeoutMs?: number): Promise<void> {
    return this.queue.stop(timeoutMs)
  }

  private async submit(projection: BufferedProjection): Promise<void> {
    if (projection.telemetryType === 'reconcile_outcome' && !projection.resolvedSourceEventId) {
      projection.resolvedSourceEventId = this.deriveReconcileSourceEventId(projection)
    }
    const sourceEventId =
      projection.telemetryType === 'reconcile_outcome'
        ? projection.resolvedSourceEventId
        : projection.sourceEventId
    if (!sourceEventId) throw new Error('reconcile source identity was not resolved')
    const controller = new AbortController()
    const timeout = setTimeout(() => controller.abort(), this.timeoutMs)
    try {
      const response = await this.fetchFn(
        `${this.baseUrl}/api/v1/internal/tracing/infrastructure-telemetry-events`,
        {
          method: 'POST',
          headers: {
            Authorization: `Bearer ${this.signToken()}`,
            'Content-Type': 'application/json',
          },
          body: JSON.stringify({
            events: [
              {
                sourceEventId,
                occurredAt: projection.occurredAt,
                telemetryType: projection.telemetryType,
                hostLookupReference: projection.hostLookupReference,
                ...(projection.payload ? { payload: projection.payload } : {}),
              },
            ],
          }),
          signal: controller.signal,
        }
      )
      await throwForFailedSubmit(response, 'telemetry')
    } finally {
      clearTimeout(timeout)
    }
  }
}

export function createInfrastructureTelemetryReporter(
  enabled: boolean,
  deps: ReporterDependencies
): InfrastructureTelemetryReporter | undefined {
  if (!enabled) return undefined
  return new BoundedInfrastructureTelemetryReporter(deps)
}