Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
7.6 KB
Raw Blame
import { BoundedOffPathReporter } from './boundedOffPathReporter'
import { config } from './config'
import { administrativeOutcomeReporterTotal } from './metrics'
import { throwForFailedSubmit } from './reporterHttpFailure'
import { signInternalControlJwt } from './utils/internalControlSigner'

export type AdministrativeHostOutcomeProjection = {
  sourceEventId: string
  occurredAt: string
  /**
   * `uid` is required: a Host deleted and recreated under the same name
   * restarts at generation 1, so namespace/name/generation alone can name two
   * different objects. control-api resolves the binding against the live
   * object's metadata.uid and refuses the reference without it (#694).
   */
  hostRef: { name: string; namespace: string; generation: number; uid: string }
  outcome: 'succeeded' | 'failed'
  reasonCode: string
}

export interface AdministrativeOutcomeReporter {
  enqueueHostOutcome(projection: AdministrativeHostOutcomeProjection): void
  stop(timeoutMs?: number): Promise<void>
}

export type AdministrativeOutcomeReporterDependencies = {
  baseUrl?: string
  signToken?: () => string
  fetchFn?: typeof fetch
  capacity?: number
  retryLimit?: number
  timeoutMs?: number
  random?: () => number
  dedupeCapacity?: number
}

const DEFAULT_CAPACITY = 64
const DEFAULT_RETRY_LIMIT = 2
const DEFAULT_TIMEOUT_MS = 1_000
const DEFAULT_STOP_TIMEOUT_MS = 1_500
const DEFAULT_DEDUPE_CAPACITY = 1_024

export class BoundedAdministrativeOutcomeReporter implements AdministrativeOutcomeReporter {
  private readonly queue: BoundedOffPathReporter<AdministrativeHostOutcomeProjection>
  private readonly timeoutMs: number
  private readonly baseUrl: string
  private readonly signToken: () => string
  private readonly fetchFn: typeof fetch
  private readonly dedupeCapacity: number
  // The reconciler re-observes the same outcome on every pass (3-4 fleet
  // passes per 5 minutes) and its sourceEventId is deterministic, so each
  // fact is sent once per process (#327). A key stays here while it is queued,
  // in flight, accepted or rejected as terminal; only a drop removes it.
  // Iteration order is least recently seen first, which is the eviction
  // order. Past `dedupeCapacity` distinct keys per pass, the oldest are
  // evicted and resent; control-api answers those resends as replays.
  private readonly seen = new Set<string>()

  constructor(deps: AdministrativeOutcomeReporterDependencies) {
    const capacity = deps.capacity ?? DEFAULT_CAPACITY
    if (!Number.isSafeInteger(capacity) || capacity < 1) {
      throw new Error('administrative outcome reporter capacity must be a positive integer')
    }
    const dedupeCapacity = deps.dedupeCapacity ?? DEFAULT_DEDUPE_CAPACITY
    // Up to `capacity` keys are buffered plus one in flight. A smaller set
    // would evict a key that is still queued, and the next pass would queue
    // it a second time.
    if (!Number.isSafeInteger(dedupeCapacity) || dedupeCapacity <= capacity) {
      throw new Error(
        'administrative outcome reporter dedupeCapacity must be an integer greater than capacity'
      )
    }
    this.dedupeCapacity = dedupeCapacity
    this.timeoutMs = deps.timeoutMs ?? DEFAULT_TIMEOUT_MS
    this.baseUrl = (deps.baseUrl ?? config.controlApiBaseUrl).replace(/\/$/, '')
    this.signToken = deps.signToken ?? signInternalControlJwt
    this.fetchFn = deps.fetchFn ?? fetch
    this.queue = new BoundedOffPathReporter({
      capacity,
      retryLimit: deps.retryLimit ?? DEFAULT_RETRY_LIMIT,
      stopTimeoutMs: DEFAULT_STOP_TIMEOUT_MS,
      random: deps.random ?? Math.random,
      submit: projection => this.submit(projection),
      onEnqueued: () => administrativeOutcomeReporterTotal.inc({ result: 'enqueued' }),
      onAccepted: () => administrativeOutcomeReporterTotal.inc({ result: 'accepted' }),
      onTerminal: (_projection, result) => administrativeOutcomeReporterTotal.inc({ result }),
      onDrop: (projection, reason) => {
        // A dropped outcome was never delivered; forgetting it lets the next
        // reconcile pass enqueue it again, so no outcome is lost.
        this.seen.delete(projection.sourceEventId)
        administrativeOutcomeReporterTotal.inc({ result: reason })
      },
    })
  }

  /**
   * Deduplicates on the `sourceEventId` this reporter is handed, which it
   * treats as opaque. Its caller builds it without the `reasonCode`
   * (hostReconciler.ts, `enqueueAdministrativeOutcome`), so for one operation,
   * generation, Host uid and outcome the first failure wins and later ones are
   * deduplicated. That matches the server, which hashes the `reasonCode` into
   * the stored row: a second `failed` with another reason under the same key
   * would come back 409 and never be stored either. The per-reason detail is
   * not lost — `controller_error` keeps it in full, with a unique occurrence id
   * per event (#696).
   *
   * `seen` is a bounded LRU, so "the first one wins" holds only while the key
   * is still resident. A key evicted under load is enqueued again and refused
   * by the server as a 409 instead, which this reporter treats as terminal.
   */
  enqueueHostOutcome(projection: AdministrativeHostOutcomeProjection): void {
    const { sourceEventId } = projection
    if (this.seen.delete(sourceEventId)) {
      // Re-inserted as most recently seen, so a key observed on every pass is
      // not evicted behind keys observed once.
      this.seen.add(sourceEventId)
      administrativeOutcomeReporterTotal.inc({ result: 'deduplicated' })
      return
    }
    if (this.seen.size >= this.dedupeCapacity) {
      const oldest = this.seen.values().next().value
      if (oldest !== undefined) this.seen.delete(oldest)
    }
    this.seen.add(sourceEventId)
    this.queue.enqueue({ ...projection, hostRef: { ...projection.hostRef } })
  }

  stop(timeoutMs?: number): Promise<void> {
    return this.queue.stop(timeoutMs)
  }

  private async submit(projection: AdministrativeHostOutcomeProjection): Promise<void> {
    const controller = new AbortController()
    const timeout = setTimeout(() => controller.abort(), this.timeoutMs)
    try {
      const response = await this.fetchFn(
        `${this.baseUrl}/api/v1/internal/tracing/administrative-events`,
        {
          method: 'POST',
          headers: {
            Authorization: `Bearer ${this.signToken()}`,
            'Content-Type': 'application/json',
          },
          body: JSON.stringify({
            events: [
              {
                sourceEventId: projection.sourceEventId,
                occurredAt: projection.occurredAt,
                kind: 'linked_outcome',
                reasonCode: projection.reasonCode,
                // Parsed by STATUS_REF in
                // control-api/src/services/tracing/adminOperationBindingResolver.ts.
                // The two packages cannot import each other, so each side pins
                // the same example string in its tests (#694).
                sourceStatusRef: `host:${projection.hostRef.namespace}/${projection.hostRef.name}:generation=${projection.hostRef.generation}:uid=${projection.hostRef.uid}`,
                payload: {
                  resource_class: 'Host',
                  status: projection.outcome,
                },
              },
            ],
          }),
          signal: controller.signal,
        }
      )
      await throwForFailedSubmit(response, 'administrative outcome')
    } finally {
      clearTimeout(timeout)
    }
  }
}

export function createAdministrativeOutcomeReporter(
  enabled: boolean,
  deps: AdministrativeOutcomeReporterDependencies
): AdministrativeOutcomeReporter | undefined {
  if (!enabled) return undefined
  return new BoundedAdministrativeOutcomeReporter(deps)
}