Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
14.5 KB
Raw Blame
import { rootLogger } from './observability/logger.js'
import {
  canonicalTracingClusterLocation,
  canonicalTracingClusterName,
  canonicalTracingEnvironment,
} from './services/tracing/environment.js'
import {
  InfrastructureTelemetryEventService,
  type TraceMaintenanceInfrastructurePrincipal,
} from './services/tracing/infrastructureTelemetryEvents.js'
import {
  gcpBigQueryBillingConfigFromEnv,
  loadNormalizedGcpBillingRows,
} from './services/tracing/maintenance/gcpBigQueryBillingAdapter.js'
import {
  gcpBigQueryPricingConfigFromEnv,
  loadNormalizedGcpPricingEvidence,
} from './services/tracing/maintenance/gcpBigQueryPricingAdapter.js'
import { appendInventorySnapshotInChunks } from './services/tracing/maintenance/inventoryAppender.js'
import { createControlPlaneInventoryCache } from './services/tracing/maintenance/inventorySampler.js'
import {
  type MaintenanceCostPersistenceConfig,
  closedUtcDayRange,
  runMaintenanceCostPersistence,
} from './services/tracing/maintenance/maintenanceCostService.js'
import { runRetentionBatch } from './services/tracing/maintenance/retention.js'
import {
  TraceMaintenanceShutdownCoordinator,
  type TraceMaintenanceShutdownSignal,
  waitForAbortableDelay,
} from './services/tracing/maintenance/workerLifecycle.js'
import { closeTraceMaintenancePool, withTraceMaintenanceClient } from './services/tracing/pools.js'

const WAKE_INTERVAL_MS = 60_000
const SHUTDOWN_TIMEOUT_MS = 25_000
const GCP_EVIDENCE_ERROR_RETRY_MS = 900_000
const DEFAULT_COST_ROLLUP_INTERVAL_MS = 3_600_000

const maintenancePrincipal: TraceMaintenanceInfrastructurePrincipal = {
  kind: 'trace_maintenance',
  sourceService: 'control-api',
  serviceSub: 'trace-maintenance-worker',
  credentialId: 'in-process',
  resourceAuthority: 'control_plane_inventory',
  allowedTelemetryTypes: ['capacity_sample'],
}

const inventory = createControlPlaneInventoryCache()
let wakeInFlight = false
let nextGcpBillingImportAt = 0
let nextGcpPricingImportAt = 0
let nextCostRollupAt = 0

function boundedEnvInteger(input: {
  name: string
  fallback: number
  minimum: number
  maximum: number
}): number {
  const value = process.env[input.name]?.trim()
  if (!value) return input.fallback
  const parsed = Number(value)
  if (!Number.isSafeInteger(parsed) || parsed < input.minimum || parsed > input.maximum) {
    throw new Error(
      `${input.name} must be an integer between ${input.minimum} and ${input.maximum}`
    )
  }
  return parsed
}

function costPersistenceConfig(): MaintenanceCostPersistenceConfig {
  const enabled = process.env.TRACING_INFRASTRUCTURE_COST_ENABLED === 'true'
  return {
    enabled,
    cloudProjectId: process.env.GCP_PROJECT_ID?.trim() ?? '',
    clusterLocation: enabled ? canonicalTracingClusterLocation() : '',
    clusterClass: process.env.KUBERNETES_CLUSTER_CLASS?.trim() ?? '',
    currency: process.env.TRACING_INFRASTRUCTURE_COST_CURRENCY?.trim() ?? 'USD',
    requestedCapacityLookbackDays: boundedEnvInteger({
      name: 'TRACING_INFRASTRUCTURE_COST_LOOKBACK_DAYS',
      fallback: 7,
      minimum: 1,
      maximum: 31,
    }),
    requestedCapacityFinalizationDelayHours: boundedEnvInteger({
      name: 'TRACING_INFRASTRUCTURE_COST_FINALIZATION_DELAY_HOURS',
      fallback: 24,
      minimum: 1,
      maximum: 168,
    }),
  }
}

async function loadBilledCostEvidence(now: Date, costConfig: MaintenanceCostPersistenceConfig) {
  if (!costConfig.enabled) {
    return {
      rows: [],
      finalizationDelayHours: 96,
      outcome: 'disabled' as const,
      exportLagSeconds: null,
    }
  }
  try {
    const config = gcpBigQueryBillingConfigFromEnv({
      target: {
        cloudProjectId: costConfig.cloudProjectId,
        clusterLocation: costConfig.clusterLocation,
        clusterName: canonicalTracingClusterName(),
        environment: canonicalTracingEnvironment(),
      },
    })
    if (!config.enabled || now.getTime() < nextGcpBillingImportAt) {
      return {
        rows: [],
        finalizationDelayHours: config.enabled ? config.finalizationDelayHours : 96,
        outcome: config.enabled ? ('cadence' as const) : ('disabled' as const),
        exportLagSeconds: null,
      }
    }
    nextGcpBillingImportAt = now.getTime() + config.importIntervalMs
    const range = closedUtcDayRange(now, config.lookbackDays)
    const rows = await loadNormalizedGcpBillingRows({
      config,
      ...range,
      now,
    })
    const latestWatermark = rows
      .map(row => Date.parse(row.exportWatermark))
      .filter(Number.isFinite)
      .sort((left, right) => right - left)[0]
    return {
      rows,
      finalizationDelayHours: config.finalizationDelayHours,
      outcome: rows.length === 0 ? ('empty' as const) : ('success' as const),
      exportLagSeconds:
        latestWatermark === undefined
          ? null
          : Math.max(0, (now.getTime() - latestWatermark) / 1_000),
    }
  } catch (error) {
    nextGcpBillingImportAt = now.getTime() + GCP_EVIDENCE_ERROR_RETRY_MS
    rootLogger.error({ err: error }, 'normalized GCP billing import failed')
    return {
      rows: [],
      finalizationDelayHours: 96,
      outcome: 'error' as const,
      exportLagSeconds: null,
    }
  }
}

async function loadPriceEvidence(now: Date, costConfig: MaintenanceCostPersistenceConfig) {
  if (!costConfig.enabled) return { evidence: [], outcome: 'disabled' as const }
  try {
    const config = gcpBigQueryPricingConfigFromEnv({
      target: {
        cloudProjectId: costConfig.cloudProjectId,
        region: costConfig.clusterLocation,
        clusterClass: costConfig.clusterClass,
        currency: costConfig.currency,
      },
    })
    if (!config.enabled || now.getTime() < nextGcpPricingImportAt) {
      return {
        evidence: [],
        outcome: config.enabled ? ('cadence' as const) : ('disabled' as const),
      }
    }
    nextGcpPricingImportAt = now.getTime() + config.importIntervalMs
    const evidence = await loadNormalizedGcpPricingEvidence({ config, now })
    return {
      evidence,
      outcome: evidence.length === 0 ? ('empty' as const) : ('success' as const),
    }
  } catch (error) {
    nextGcpPricingImportAt = now.getTime() + GCP_EVIDENCE_ERROR_RETRY_MS
    rootLogger.error({ err: error }, 'normalized GCP pricing import failed')
    return { evidence: [], outcome: 'error' as const }
  }
}

async function appendInventory(client: import('./db.js').DbClient, now: Date): Promise<number> {
  const snapshot = inventory.snapshot(now)
  if (snapshot.workloads.length === 0) return 0
  const service = new InfrastructureTelemetryEventService({
    transaction: async () => {
      throw new Error('maintenance append requires caller transaction')
    },
    now: () => now,
  })
  return appendInventorySnapshotInChunks({
    client,
    service,
    principal: maintenancePrincipal,
    snapshot,
    now,
    environment: canonicalTracingEnvironment(),
    clusterName: canonicalTracingClusterName(),
  })
}

async function inspectStreamRegistrationGaps(
  client: import('./db.js').DbClient
): Promise<Record<string, number>> {
  // This is the family-to-stream transaction invariant, not producer ordering.
  // Postgres sequences may have legitimate holes; producer reporters own
  // drop/gap signals and validation gates use independent expected occurrences.
  const result = await client.query(`
    SELECT 'agent_run' AS family, COUNT(*)::int AS gap_count
      FROM (SELECT event_id FROM agent_run_events ORDER BY ingest_sequence DESC LIMIT 1000) event
      LEFT JOIN governed_event_stream stream
        ON stream.event_family = 'agent_run' AND stream.event_id = event.event_id
     WHERE stream.event_id IS NULL
    UNION ALL
    SELECT 'administrative', COUNT(*)::int
      FROM (SELECT event_id FROM administrative_events ORDER BY ingest_sequence DESC LIMIT 1000) event
      LEFT JOIN governed_event_stream stream
        ON stream.event_family = 'administrative' AND stream.event_id = event.event_id
     WHERE stream.event_id IS NULL
    UNION ALL
    SELECT 'infrastructure_telemetry', COUNT(*)::int
      FROM (SELECT event_id FROM infrastructure_telemetry_events ORDER BY ingest_sequence DESC LIMIT 1000) event
      LEFT JOIN governed_event_stream stream
        ON stream.event_family = 'infrastructure_telemetry' AND stream.event_id = event.event_id
     WHERE stream.event_id IS NULL
  `)
  const gaps: Record<string, number> = {}
  for (const row of result.rows as Array<{ family: string; gap_count: number }>) {
    gaps[row.family] = Number(row.gap_count)
  }
  return gaps
}

async function wake(): Promise<void> {
  if (wakeInFlight) {
    rootLogger.warn({ outcome: 'overlap_prevented' }, 'trace maintenance wake overlap prevented')
    return
  }
  wakeInFlight = true
  const startedAt = Date.now()
  let outcome = 'failed'
  let costRollupAttempted = false
  try {
    const now = new Date()
    const costConfig = costPersistenceConfig()
    const costRollupIntervalMs = boundedEnvInteger({
      name: 'TRACING_INFRASTRUCTURE_COST_ROLLUP_INTERVAL_MS',
      fallback: DEFAULT_COST_ROLLUP_INTERVAL_MS,
      minimum: 300_000,
      maximum: 86_400_000,
    })
    costRollupAttempted = costConfig.enabled && now.getTime() >= nextCostRollupAt
    if (costRollupAttempted) nextCostRollupAt = now.getTime() + costRollupIntervalMs
    // Remote cost evidence is fetched before any Postgres checkout or transaction.
    const [gcpBillingEvidence, gcpPricingEvidence] = costRollupAttempted
      ? await Promise.all([
          loadBilledCostEvidence(now, costConfig),
          loadPriceEvidence(now, costConfig),
        ])
      : [
          {
            rows: [],
            finalizationDelayHours: 96,
            outcome: costConfig.enabled ? ('cadence' as const) : ('disabled' as const),
            exportLagSeconds: null,
          },
          {
            evidence: [],
            outcome: costConfig.enabled ? ('cadence' as const) : ('disabled' as const),
          },
        ]
    await withTraceMaintenanceClient(async client => {
      try {
        await client.query('BEGIN')
        const lock = await client.query(
          "SELECT pg_try_advisory_xact_lock(hashtext('governed-trace-maintenance-v1')) AS acquired"
        )
        if (!lock.rows[0]?.acquired) {
          outcome = 'lock_not_acquired'
          await client.query('ROLLBACK')
          rootLogger.info(
            { outcome, durationMs: Date.now() - startedAt },
            'trace maintenance wake skipped because the advisory lock was not acquired'
          )
          return
        }
        await client.query("SELECT set_config('statement_timeout', '30000', true)")
        const streamRegistrationGaps = await inspectStreamRegistrationGaps(client)
        if (Object.values(streamRegistrationGaps).some(count => count > 0)) {
          rootLogger.error(
            { streamRegistrationGaps },
            'governed trace stream registration invariant violated'
          )
        }
        const inventoryEvents = await appendInventory(client, now)
        const costPersistence = costRollupAttempted
          ? await runMaintenanceCostPersistence({
              db: client,
              now,
              config: costConfig,
              normalizedGcpBillingRows: gcpBillingEvidence.rows,
              gcpBillingFinalizationDelayHours: gcpBillingEvidence.finalizationDelayHours,
              priceSnapshots: gcpPricingEvidence.evidence,
            })
          : {
              priceSnapshotsInserted: 0,
              requestedCapacityVersions: 0,
              billedVersions: 0,
              skippedReason: costConfig.enabled ? 'cadence' : 'disabled',
            }
        const result = await runRetentionBatch(client)
        await client.query('COMMIT')
        outcome = 'completed'
        rootLogger.info(
          {
            ...result,
            inventoryEvents,
            costPersistence,
            streamRegistrationGaps,
            gcpBilling: {
              outcome: gcpBillingEvidence.outcome,
              rowCount: gcpBillingEvidence.rows.length,
              exportLagSeconds: gcpBillingEvidence.exportLagSeconds,
            },
            gcpPricing: {
              outcome: gcpPricingEvidence.outcome,
              snapshotCount: gcpPricingEvidence.evidence.length,
            },
            durationMs: Date.now() - startedAt,
          },
          'trace maintenance wake completed'
        )
      } catch (error) {
        await client.query('ROLLBACK').catch(() => undefined)
        throw error
      }
    })
  } catch (error) {
    if (costRollupAttempted) {
      const retryAt = Date.now() + GCP_EVIDENCE_ERROR_RETRY_MS
      nextCostRollupAt = Math.min(nextCostRollupAt, retryAt)
      nextGcpBillingImportAt = Math.min(nextGcpBillingImportAt, retryAt)
      nextGcpPricingImportAt = Math.min(nextGcpPricingImportAt, retryAt)
    }
    rootLogger.error(
      { err: error, outcome, durationMs: Date.now() - startedAt },
      'trace maintenance wake failed'
    )
  } finally {
    wakeInFlight = false
  }
}

async function main(): Promise<void> {
  const shutdown = new TraceMaintenanceShutdownCoordinator({
    timeoutMs: SHUTDOWN_TIMEOUT_MS,
    onRequested: signal => {
      rootLogger.info({ signal }, 'trace maintenance shutdown requested')
    },
    onCompleted: signal => {
      rootLogger.info({ signal, outcome: 'completed' }, 'trace maintenance shutdown completed')
    },
    onTimedOut: (signal, timeoutMs) => {
      rootLogger.error(
        { signal, timeoutMs, outcome: 'timeout' },
        'trace maintenance shutdown timed out with a wake still in flight'
      )
    },
    exit: code => process.exit(code),
  })
  const requestShutdown = (signal: TraceMaintenanceShutdownSignal) => {
    shutdown.request(signal)
  }
  const onSigterm = () => requestShutdown('SIGTERM')
  const onSigint = () => requestShutdown('SIGINT')
  process.once('SIGTERM', onSigterm)
  process.once('SIGINT', onSigint)

  let inventoryStarted = false
  try {
    await inventory.start()
    inventoryStarted = true
    while (!shutdown.signal.aborted) {
      await wake()
      if (!shutdown.signal.aborted) {
        await waitForAbortableDelay(WAKE_INTERVAL_MS, shutdown.signal)
      }
    }
  } finally {
    const cleanup = await Promise.allSettled([
      inventoryStarted ? inventory.stop() : Promise.resolve(),
      closeTraceMaintenancePool(),
    ])
    const failedCleanup = cleanup.find(result => result.status === 'rejected')
    process.off('SIGTERM', onSigterm)
    process.off('SIGINT', onSigint)
    if (failedCleanup?.status === 'rejected') throw failedCleanup.reason
    shutdown.finish()
  }
}

void main().catch(error => {
  rootLogger.error({ err: error }, 'trace maintenance worker terminated unexpectedly')
  process.exitCode = 1
})