Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
19.6 KB
Raw Blame
import { describe, expect, it, vi } from 'vitest'
import type { V1Deployment } from '@kubernetes/client-node'
import type {
  DailyCostKey,
  NormalizedGcpBillingRow,
  PersistedDailyCostVersion,
} from '../src/services/tracing/maintenance/contracts.js'
import { reconcileNormalizedGcpBilling } from '../src/services/tracing/maintenance/gcpBillingReconciliation.js'
import { appendInventorySnapshotInChunks } from '../src/services/tracing/maintenance/inventoryAppender.js'
import {
  AllowlistedDeploymentInventoryCache,
  type DeploymentInventorySource,
  deploymentToInventoryWorkload,
} from '../src/services/tracing/maintenance/inventorySampler.js'
import { selectPeriodCosts } from '../src/services/tracing/maintenance/periodSelection.js'
import { estimateRequestedCapacityDaily } from '../src/services/tracing/maintenance/requestedCapacityEstimate.js'
import { runRetentionBatch } from '../src/services/tracing/maintenance/retention.js'

const key: DailyCostKey = {
  utcDay: '2026-07-10',
  cloudProvider: 'gcp',
  cloudProjectId: 'project-1',
  clusterLocation: 'europe-west1',
  clusterName: 'cluster-1',
  environment: 'test',
  namespace: 'control-plane',
  workloadKind: 'Deployment',
  workloadRef: 'control-api',
  currency: 'USD',
}
const sha = 'a'.repeat(64)

type FakeInformerHandler = () => void

class FakeDeploymentInformer {
  private readonly handlers = new Map<string, FakeInformerHandler[]>()

  constructor(private readonly deployments: readonly V1Deployment[]) {}

  on(event: string, handler: FakeInformerHandler): void {
    this.handlers.set(event, [...(this.handlers.get(event) ?? []), handler])
  }

  async start(): Promise<void> {
    for (const handler of this.handlers.get('connect') ?? []) handler()
  }

  async stop(): Promise<void> {}

  list(namespace?: string): readonly V1Deployment[] {
    return this.deployments.filter(deployment =>
      namespace ? deployment.metadata?.namespace === namespace : true
    )
  }
}

function deployment(input: {
  namespace: string
  name: string
  labels?: Record<string, string>
  cpu?: string
  memory?: string
  replicas?: number
  observedReplicas?: number
  readyReplicas?: number
  resourceVersion?: string
  uid?: string
  generation?: number
  cpuLimit?: string
  memoryLimit?: string
}): V1Deployment {
  return {
    metadata: {
      name: input.name,
      namespace: input.namespace,
      labels: input.labels,
      resourceVersion: input.resourceVersion,
      uid: input.uid,
      generation: input.generation,
    },
    spec: {
      replicas: input.replicas,
      template: {
        metadata: {},
        spec: {
          containers: [
            {
              name: 'main',
              image: 'test',
              resources: {
                requests: { cpu: input.cpu, memory: input.memory },
                limits: { cpu: input.cpuLimit, memory: input.memoryLimit },
              },
            },
          ],
        },
      },
    },
    status: { replicas: input.observedReplicas, readyReplicas: input.readyReplicas },
  }
}

function source(input: {
  namespace: string
  deployments: readonly V1Deployment[]
  include: DeploymentInventorySource['include']
  requiredNames?: readonly string[]
}): DeploymentInventorySource {
  return {
    namespace: input.namespace,
    informer: new FakeDeploymentInformer(
      input.deployments
    ) as DeploymentInventorySource['informer'],
    include: input.include,
    requiredNames: input.requiredNames ? new Set(input.requiredNames) : undefined,
  }
}

describe('trace maintenance cost contracts', () => {
  it('chunks 101 inventory appends at 100 while preserving full count and deterministic order', async () => {
    const appendManyInTransaction = vi.fn().mockResolvedValue([])
    const client = { query: vi.fn() } as never
    const workloads = Array.from({ length: 101 }, (_, index) => ({
      namespace: 'mcp-server' as const,
      workloadKind: 'Deployment' as const,
      workloadRef: `worker-${String(index).padStart(3, '0')}`,
      kubernetesUid: `uid-${index}`,
      metadataGeneration: index + 1,
      desiredReplicas: 1,
      observedReplicas: 1,
      readyReplicas: index % 2,
      cpuRequestNanoCores: 100_000_000n + BigInt(index),
      cpuLimitNanoCores: 200_000_000n + BigInt(index),
      memoryRequestBytes: 134_217_728n + BigInt(index),
      memoryLimitBytes: 268_435_456n + BigInt(index),
      stableLabels: {},
    }))

    await expect(
      appendInventorySnapshotInChunks({
        client,
        service: { appendManyInTransaction } as never,
        principal: {
          kind: 'trace_maintenance',
          sourceService: 'control-api',
          serviceSub: 'trace-maintenance-worker',
          credentialId: 'in-process',
          resourceAuthority: 'control_plane_inventory',
          allowedTelemetryTypes: ['capacity_sample'],
        },
        snapshot: {
          observedAt: '2026-07-10T12:00:30.000Z',
          resourceVersion: 'rv-101',
          complete: true,
          workloads,
          omittedAllowlistedWorkloads: [],
        },
        now: new Date('2026-07-10T12:00:30.123Z'),
        environment: 'test',
        clusterName: 'cluster-1',
      })
    ).resolves.toBe(101)

    expect(appendManyInTransaction).toHaveBeenCalledTimes(2)
    expect(appendManyInTransaction.mock.calls.map(call => call[2])).toHaveLength(2)
    expect(appendManyInTransaction.mock.calls[0]![2]).toHaveLength(100)
    expect(appendManyInTransaction.mock.calls[1]![2]).toHaveLength(1)
    const appended = appendManyInTransaction.mock.calls.flatMap(call => call[2])
    expect(appended.map(entry => entry.binding.workloadRef)).toEqual(
      workloads.map(workload => `${workload.namespace}/${workload.workloadRef}`)
    )
    expect(new Set(appended.map(entry => entry.input.sourceEventId)).size).toBe(101)
    expect(appended[0]!.binding).toMatchObject({
      environment: 'test',
      clusterName: 'cluster-1',
      workloadRef: 'mcp-server/worker-000',
      kubernetesUid: 'uid-0',
      metadataGeneration: 1,
    })
    expect(appended[0]!.input).toMatchObject({
      observedReplicas: 1,
      cpuLimitCores: 0.2,
      memoryLimitBytes: 268_435_456,
    })
    expect(appended.at(-1)!.binding.workloadRef).toBe('mcp-server/worker-100')
  })

  it('runs bounded retention through database retention functions only', async () => {
    const query = vi
      .fn()
      .mockResolvedValueOnce({ rows: [], rowCount: 1 })
      .mockResolvedValueOnce({ rows: [], rowCount: 2 })
      .mockResolvedValueOnce({ rows: [], rowCount: 3 })
      .mockResolvedValueOnce({ rows: [], rowCount: 4 })
      .mockResolvedValueOnce({ rows: [], rowCount: 5 })
    await expect(runRetentionBatch({ query }, new Date('2026-07-11T00:00:00Z'))).resolves.toEqual({
      eventsDeleted: 6,
      costsDeleted: 4,
      promptsDeleted: 5,
      saturatedGrains: [],
    })
    expect(query).toHaveBeenCalledTimes(5)
    expect(query.mock.calls.map(([sql]) => String(sql))).toEqual([
      'SELECT event_id FROM governed_trace_prune_expired_events($1, $2)',
      'SELECT event_id FROM governed_trace_prune_expired_events($1, $2)',
      'SELECT event_id FROM governed_trace_prune_expired_events($1, $2)',
      'SELECT id FROM governed_trace_prune_expired_costs($1)',
      'SELECT approval_request_id FROM governed_trace_prune_expired_prompts($1)',
    ])
    expect(query.mock.calls.map(([, params]) => params)).toEqual([
      ['agent_run', 200],
      ['administrative', 200],
      ['infrastructure_telemetry', 200],
      [200],
      [200],
    ])
    expect(query.mock.calls.map(([sql]) => String(sql)).join('\n')).not.toMatch(
      /set_config|DELETE/i
    )
  })

  it('reclaims unused grain capacity without exceeding the 1000-row wake budget', async () => {
    const query = vi
      .fn()
      .mockResolvedValueOnce({ rows: [], rowCount: 200 })
      .mockResolvedValueOnce({ rows: [], rowCount: 12 })
      .mockResolvedValueOnce({ rows: [], rowCount: 200 })
      .mockResolvedValueOnce({ rows: [], rowCount: 200 })
      .mockResolvedValueOnce({ rows: [], rowCount: 0 })
      .mockResolvedValueOnce({ rows: [], rowCount: 200 })
      .mockResolvedValueOnce({ rows: [], rowCount: 188 })

    await expect(runRetentionBatch({ query })).resolves.toEqual({
      eventsDeleted: 800,
      costsDeleted: 200,
      promptsDeleted: 0,
      saturatedGrains: ['agent_run', 'infrastructure_telemetry', 'infrastructure_cost'],
    })
    expect(query.mock.calls.map(([, params]) => params)).toEqual([
      ['agent_run', 200],
      ['administrative', 200],
      ['infrastructure_telemetry', 200],
      [200],
      [200],
      ['agent_run', 200],
      ['infrastructure_telemetry', 188],
    ])
    expect(query.mock.calls.map(([sql]) => String(sql)).join('\n')).not.toMatch(/COUNT\s*\(/i)
  })

  it('uses the full bounded wake budget for a single backlogged grain', async () => {
    const query = vi
      .fn()
      .mockResolvedValueOnce({ rows: [], rowCount: 200 })
      .mockResolvedValueOnce({ rows: [], rowCount: 0 })
      .mockResolvedValueOnce({ rows: [], rowCount: 0 })
      .mockResolvedValueOnce({ rows: [], rowCount: 0 })
      .mockResolvedValueOnce({ rows: [], rowCount: 0 })
      .mockResolvedValueOnce({ rows: [], rowCount: 200 })
      .mockResolvedValueOnce({ rows: [], rowCount: 200 })
      .mockResolvedValueOnce({ rows: [], rowCount: 200 })
      .mockResolvedValueOnce({ rows: [], rowCount: 200 })

    await expect(runRetentionBatch({ query })).resolves.toEqual({
      eventsDeleted: 1_000,
      costsDeleted: 0,
      promptsDeleted: 0,
      saturatedGrains: ['agent_run'],
    })
    expect(query).toHaveBeenCalledTimes(9)
    expect(query.mock.calls.reduce((sum, [, params]) => sum + Number(params?.at(-1)), 0)).toBe(
      1_800
    )
  })

  it('normalizes Kubernetes resource quantities and strips high-cardinality labels', () => {
    const workload = deploymentToInventoryWorkload({
      metadata: {
        name: 'control-api',
        namespace: 'control-plane',
        uid: 'deployment-uid',
        generation: 9,
        labels: {
          app: 'control-api',
          'clerum.io/managed-by': 'control-api',
          run_id: 'forbidden',
          'clerum.io/human': 'forbidden',
          'clerum.io/team': 'forbidden',
        },
      },
      spec: {
        replicas: 2,
        template: {
          metadata: {},
          spec: {
            containers: [
              {
                name: 'api',
                image: 'test',
                resources: {
                  requests: { cpu: '500m', memory: '1.5Gi' },
                  limits: { cpu: '2', memory: '2Gi' },
                },
              },
            ],
          },
        },
      },
      status: { replicas: 2, readyReplicas: 1 },
    })
    expect(workload).toMatchObject({
      cpuRequestNanoCores: 500_000_000n,
      cpuLimitNanoCores: 2_000_000_000n,
      memoryRequestBytes: 1_610_612_736n,
      memoryLimitBytes: 2_147_483_648n,
      kubernetesUid: 'deployment-uid',
      metadataGeneration: 9,
      observedReplicas: 2,
      stableLabels: { app: 'control-api', 'clerum.io/managed-by': 'control-api' },
    })
  })

  it('samples static control-plane plus HCC and WRC managed Deployments by namespace', async () => {
    const cache = AllowlistedDeploymentInventoryCache.forSources([
      source({
        namespace: 'control-plane',
        deployments: [
          deployment({
            namespace: 'control-plane',
            name: 'control-api',
            labels: { app: 'control-api' },
            cpu: '250m',
            memory: '256Mi',
            replicas: 2,
            readyReplicas: 2,
            resourceVersion: '10',
          }),
          deployment({
            namespace: 'control-plane',
            name: 'foreign-control',
            labels: { app: 'foreign-control' },
          }),
        ],
        requiredNames: ['control-api'],
        include: item => item.metadata?.name === 'control-api',
      }),
      source({
        namespace: 'mcp-server',
        deployments: [
          deployment({
            namespace: 'mcp-server',
            name: 'chatllm',
            labels: {
              'clerum.io/managed-by': 'host-context-controller',
              'clerum.io/team': 'forbidden',
            },
            cpu: '1',
            memory: '1Gi',
            replicas: 1,
            readyReplicas: 1,
            resourceVersion: '20',
          }),
          deployment({
            namespace: 'mcp-server',
            name: 'manual',
            labels: { 'clerum.io/managed-by': 'operator' },
          }),
        ],
        include: item =>
          item.metadata?.labels?.['clerum.io/managed-by'] === 'host-context-controller',
      }),
      source({
        namespace: 'sandbox-recipes',
        deployments: [
          deployment({
            namespace: 'sandbox-recipes',
            name: 'workflow-child',
            labels: {
              'clerum.io/managed-by': 'wrc',
              'clerum.io/human': 'forbidden',
              run_id: 'forbidden',
            },
            cpu: '750m',
            memory: '128Mi',
            replicas: 3,
            readyReplicas: 2,
            resourceVersion: '30',
          }),
          deployment({
            namespace: 'sandbox-recipes',
            name: 'hcc-network-helper',
            labels: { 'clerum.io/managed-by': 'host-context-controller' },
          }),
        ],
        include: item => item.metadata?.labels?.['clerum.io/managed-by'] === 'wrc',
      }),
    ])

    await cache.start()
    const snapshot = cache.snapshot(new Date('2026-07-10T12:00:00Z'))

    expect(snapshot).toMatchObject({
      complete: true,
      omittedAllowlistedWorkloads: [],
      resourceVersion:
        'control-plane/control-api@10:mcp-server/chatllm@20:sandbox-recipes/workflow-child@30',
    })
    expect(
      snapshot.workloads.map(workload => `${workload.namespace}/${workload.workloadRef}`)
    ).toEqual(['control-plane/control-api', 'mcp-server/chatllm', 'sandbox-recipes/workflow-child'])
    expect(snapshot.workloads.map(workload => workload.cpuRequestNanoCores)).toEqual([
      250_000_000n,
      1_000_000_000n,
      750_000_000n,
    ])
    expect(snapshot.workloads.map(workload => workload.memoryRequestBytes)).toEqual([
      268_435_456n,
      1_073_741_824n,
      134_217_728n,
    ])
    expect(snapshot.workloads[1]!.stableLabels).toEqual({
      'clerum.io/managed-by': 'host-context-controller',
    })
    expect(snapshot.workloads[2]!.stableLabels).toEqual({ 'clerum.io/managed-by': 'wrc' })
  })

  it('marks inventory incomplete when a namespace informer is not connected or allowlisted work is missing', async () => {
    const cache = AllowlistedDeploymentInventoryCache.forSources([
      source({
        namespace: 'control-plane',
        deployments: [],
        requiredNames: ['control-api'],
        include: item => item.metadata?.name === 'control-api',
      }),
      source({
        namespace: 'mcp-server',
        deployments: [
          deployment({
            namespace: 'mcp-server',
            name: 'chatllm',
            labels: { 'clerum.io/managed-by': 'host-context-controller' },
          }),
        ],
        include: item =>
          item.metadata?.labels?.['clerum.io/managed-by'] === 'host-context-controller',
      }),
    ])

    const snapshot = cache.snapshot(new Date('2026-07-10T12:00:00Z'))

    expect(snapshot.complete).toBe(false)
    expect(snapshot.omittedAllowlistedWorkloads).toEqual(['control-plane/control-api'])
    expect(
      snapshot.workloads.map(workload => `${workload.namespace}/${workload.workloadRef}`)
    ).toEqual(['mcp-server/chatllm'])
  })

  it('values requested CPU and memory with immutable selected snapshots', () => {
    const result = estimateRequestedCapacityDaily({
      key,
      clusterClass: 'gke-standard',
      intervalStart: '2026-07-10T00:00:00Z',
      intervalEnd: '2026-07-10T01:00:00Z',
      asOfUtc: '2026-07-10T02:00:00Z',
      desiredReplicas: 2,
      cpuRequestNanoCores: 1_000_000_000n,
      memoryRequestBytes: 1_073_741_824n,
      publicationState: 'provisional',
      completenessStatus: 'complete',
      sourceCount: 1,
      sourceSha256: sha,
      priceSnapshots: [
        {
          id: 'cpu-v1',
          cloudProvider: 'gcp',
          cloudProjectId: 'project-1',
          region: 'europe-west1',
          clusterClass: 'gke-standard',
          resourceClass: 'cpu',
          unit: 'vCPU_hour',
          unitPrice: '1.000000000',
          currency: 'USD',
          effectiveFrom: '2026-01-01T00:00:00Z',
          sourceRef: 'fixture',
          sourceSha256: sha,
        },
        {
          id: 'memory-v1',
          cloudProvider: 'gcp',
          cloudProjectId: 'project-1',
          region: 'europe-west1',
          clusterClass: 'gke-standard',
          resourceClass: 'memory',
          unit: 'GiB_hour',
          unitPrice: '0.500000000',
          currency: 'USD',
          effectiveFrom: '2026-01-01T00:00:00Z',
          sourceRef: 'fixture',
          sourceSha256: sha,
        },
      ],
    })
    expect(result).toMatchObject({
      valuationKind: 'estimated',
      selectedBasis: 'requested_capacity',
      grossAmount: '3.000000000',
    })
  })

  it('preserves GCP credits and allocation buckets with exact conservation', () => {
    const row = (
      amount: string,
      costType: NormalizedGcpBillingRow['costType'],
      allocationStatus: NormalizedGcpBillingRow['allocationStatus']
    ): NormalizedGcpBillingRow => ({
      schemaVersion: 'gcp-billing-v1',
      usageUtcDay: key.utcDay,
      cloudProjectId: key.cloudProjectId,
      clusterLocation: key.clusterLocation,
      clusterName: key.clusterName,
      environment: key.environment,
      namespace: key.namespace,
      workloadKind: key.workloadKind,
      workloadRef: key.workloadRef,
      providerService: 'GKE',
      providerSku: 'compute',
      currency: key.currency,
      costType,
      allocationStatus,
      amount,
      sourceRowCount: 1,
      sourceSha256: sha,
      billingViewVersion: 'v1',
      exportWatermark: '2026-07-11T00:00:00Z',
    })
    const result = reconcileNormalizedGcpBilling({
      key,
      asOfUtc: '2026-07-11T01:00:00Z',
      publicationState: 'finalized',
      rows: [
        row('10', 'usage', 'kube:system-overhead'),
        row('-2', 'credit', 'kube:unallocated'),
        row('1', 'usage', 'goog-k8s-unsupported-sku'),
        row('-0.5', 'adjustment', 'missing-label'),
      ],
    })
    expect(result).toMatchObject({
      grossAmount: '11.000000000',
      creditsAmount: '-2.500000000',
      netAmount: '8.500000000',
    })
    expect(result.components.map(component => component.allocationBucket)).toEqual([
      'kube:system-overhead',
      'kube:unallocated',
      'unsupported',
      'adjustment',
    ])
  })

  it('selects a reproducible finalized period vector without mixing dimensions', () => {
    const daily = (utcDay: string, id: string): PersistedDailyCostVersion => ({
      id,
      rollupVersion: 1,
      predecessorVersion: null,
      key: { ...key, utcDay },
      valuationKind: 'billed',
      selectedBasis: 'gcp_request_allocation',
      publicationState: 'finalized',
      completenessStatus: 'complete',
      asOfUtc: `${utcDay}T23:59:59Z`,
      sourceIntervalStart: null,
      sourceIntervalEnd: null,
      billingExportWatermark: `${utcDay}T23:59:59Z`,
      sourceCount: 1,
      sourceSha256: sha,
      grossAmount: '2',
      creditsAmount: '-0.5',
      netAmount: '1.5',
      components: [],
    })
    const result = selectPeriodCosts('week', '2026-07-07', '2026-07-14', [
      daily('2026-07-10', 'd1'),
      daily('2026-07-11', 'd2'),
    ])
    expect(result).toMatchObject({
      publicationState: 'finalized',
      grossAmount: '4.000000000',
      creditsAmount: '-1.000000000',
      netAmount: '3.000000000',
    })
    expect(result.dailyVersionVector).toHaveLength(2)
  })
})