Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
9.6 KB
Raw Blame
import { afterAll, afterEach, beforeAll, describe, expect, it } from 'vitest'
import { randomBytes, randomUUID } from 'node:crypto'
import { existsSync } from 'node:fs'
import { mkdtemp, rm, writeFile } from 'node:fs/promises'
import { tmpdir } from 'node:os'
import { join } from 'node:path'
import { Pool } from 'pg'
import { reconcileExpiredBlobs } from '../../gfs-controller/src/db/blobReconciliation.js'
import { PgBlobStagingStore } from '../../gfs-controller/src/db/blobStaging.js'
import type { GfsMetrics } from '../../gfs-controller/src/metrics.js'
import { BlobStore } from '../../gfs-controller/src/storage/blobStore.js'
import { resolveBlobKeyPath, resolveBlobPath } from '../../gfs-controller/src/storage/paths.js'
import { initDb } from '../src/db.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'

// This suite closes the reconciler's HIGH coverage gap: reconcileExpiredBlobs is
// the sole backstop for the write/copy paths' swallowed cleanup errors, yet its
// orphan-vs-referenced predicate and its real byte deletion were only ever
// exercised against a FakeDb with a stubbed deleteByKey. Here it runs against a
// real Postgres schema + a real on-disk BlobStore, asserting business-truth:
// actual bytes on the PVC and actual manifest rows.
const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
const describeRealPostgres = adminUrl ? describe : describe.skip

function databaseUrl(baseUrl: string, database: string): string {
  const url = new URL(baseUrl)
  url.pathname = `/${database}`
  return url.toString()
}

function quoteIdent(value: string): string {
  return `"${value.replace(/"/g, '""')}"`
}

// reconcileExpiredBlobs only calls setOrphanCandidates + recordBlobCleanupFailure;
// a minimal recording stub keeps the test off prom-client while still proving the
// failure counter is never touched on the happy paths.
function metricsStub(): { metrics: GfsMetrics; failures: number } {
  const state = { failures: 0 }
  const metrics = {
    setOrphanCandidates: () => undefined,
    recordBlobCleanupFailure: () => {
      state.failures += 1
    },
  } as unknown as GfsMetrics
  return {
    metrics,
    get failures() {
      return state.failures
    },
  }
}

describeRealPostgres('GFS blob reconciliation on real PostgreSQL + on-disk BlobStore', () => {
  const database = `gfs_recon_${randomBytes(6).toString('hex')}`
  const connectionString = databaseUrl(
    adminUrl ?? 'postgresql://[email protected]/postgres',
    database
  )
  let adminPool: Pool
  let pool: Pool
  let blobRoot: string
  let blobs: BlobStore
  let manifests: PgBlobStagingStore

  // Age a just-staged manifest so it clears the reconciler's `updated_at <
  // now() - olderThanMs` gate deterministically (no sleep, no flake).
  async function ageCandidate(blobKey: string): Promise<void> {
    await pool.query(
      `UPDATE gfs_blob_manifests SET updated_at = now() - interval '1 hour' WHERE blob_key = $1`,
      [blobKey]
    )
  }

  async function manifestExists(blobKey: string): Promise<boolean> {
    const res = await pool.query('SELECT 1 FROM gfs_blob_manifests WHERE blob_key = $1', [blobKey])
    return (res.rowCount ?? 0) > 0
  }

  async function manifestState(blobKey: string): Promise<string | null> {
    const res = await pool.query('SELECT state FROM gfs_blob_manifests WHERE blob_key = $1', [
      blobKey,
    ])
    return res.rows[0]?.state ?? null
  }

  beforeAll(async () => {
    adminPool = new Pool({ connectionString: adminUrl })
    await adminPool.query(`CREATE DATABASE ${quoteIdent(database)}`)
    pool = new Pool({ connectionString })
    await initDb({ connect: () => pool.connect() })
    blobRoot = await mkdtemp(join(tmpdir(), 'gfs-recon-real-pg-'))
    blobs = new BlobStore(blobRoot, 'writer')
    manifests = new PgBlobStagingStore(pool)
  }, 60_000)

  afterEach(async () => {
    // Reconciliation scans the whole table; isolate each case's business-truth by
    // clearing manifests + resources between cases (orphaned bytes with unique
    // UUIDs left on disk cannot collide with the next case's assertions).
    await pool.query('TRUNCATE gfs_blob_manifests, gfs_resources CASCADE')
  })

  afterAll(async () => {
    try {
      await endPoolAndWaitForClients(pool)
      await rm(blobRoot, { recursive: true, force: true }).catch(() => undefined)
      if (!adminPool) return
      await adminPool.query(
        `SELECT pg_terminate_backend(pid) FROM pg_stat_activity
          WHERE datname = $1 AND pid <> pg_backend_pid()`,
        [database]
      )
      await adminPool.query(`DROP DATABASE IF EXISTS ${quoteIdent(database)}`)
    } finally {
      await adminPool?.end()
    }
  })

  it('reaps an orphaned generation blob: deletes the real bytes AND removes the manifest row', async () => {
    const resourceId = randomUUID()
    const payload = randomBytes(96)
    const written = await blobs.writeImmutable(resourceId, randomUUID(), payload)
    await manifests.recordStaged({
      blobKey: written.blobKey,
      requestId: randomUUID(),
      resourceId,
      candidateKind: 'generation',
      contentSha256: written.contentSha256,
      bytes: written.bytes,
    })
    await ageCandidate(written.blobKey)

    // Precondition: real bytes on disk, manifest row present, NO gfs_resources
    // row references this blob_key (it is a genuine orphan).
    const blobPath = resolveBlobKeyPath(blobRoot, written.blobKey)
    expect(existsSync(blobPath)).toBe(true)
    expect(await manifestExists(written.blobKey)).toBe(true)

    const recorder = metricsStub()
    const result = await reconcileExpiredBlobs(manifests, blobs, recorder.metrics, {
      olderThanMs: 1000,
      limit: 16,
    })

    expect(result.deleted).toBe(1)
    expect(result.failures).toBe(0)
    // Read the recording stub too: the reconciler's failure metric must stay
    // untouched, not just its returned tally.
    expect(recorder.failures).toBe(0)
    // Business-truth: the bytes are gone from the PVC and the manifest row is gone.
    expect(existsSync(blobPath)).toBe(false)
    expect(await manifestExists(written.blobKey)).toBe(false)
  })

  it('never deletes a generation blob still referenced by a live resource', async () => {
    const rootId = randomUUID()
    const fileId = randomUUID()
    const drive = `recon-ref-${randomBytes(4).toString('hex')}`
    const payload = randomBytes(96)
    const written = await blobs.writeImmutable(fileId, randomUUID(), payload)
    await manifests.recordStaged({
      blobKey: written.blobKey,
      requestId: randomUUID(),
      resourceId: fileId,
      candidateKind: 'generation',
      contentSha256: written.contentSha256,
      bytes: written.bytes,
    })
    // A live gfs_resources row references the blob_key — this must protect it.
    await pool.query(
      `INSERT INTO gfs_resources
         (resource_id, drive, parent_resource_id, name, kind, path_cache, version, bytes, blob_key, content_sha256)
       VALUES
         ($1,$3,NULL,'','directory','/',0,0,NULL,NULL),
         ($2,$3,$1,'ref.txt','file','/ref.txt',1,$4,$5,$6)`,
      [rootId, fileId, drive, written.bytes, written.blobKey, written.contentSha256]
    )
    await ageCandidate(written.blobKey)

    const blobPath = resolveBlobKeyPath(blobRoot, written.blobKey)
    const recorder = metricsStub()
    const result = await reconcileExpiredBlobs(manifests, blobs, recorder.metrics, {
      olderThanMs: 1000,
      limit: 16,
    })

    // The real NOT EXISTS predicate must exclude the referenced blob from the
    // delete claim, so no bytes are deleted and the bytes survive on the PVC.
    expect(result.deleted).toBe(0)
    expect(result.failures).toBe(0)
    // Read the recording stub too: the reconciler's failure metric must stay
    // untouched, not just its returned tally.
    expect(recorder.failures).toBe(0)
    // Business-truth that matters: the bytes are protected.
    expect(existsSync(blobPath)).toBe(true)
    // The now-redundant staged manifest IS metadata-GC'd by removeCommittedMetadata
    // (the gfs_resources.blob_key reference is the source of truth) — only the
    // bytes are protected, never re-deleted. This asserts the GC does NOT touch bytes.
    expect(await manifestExists(written.blobKey)).toBe(false)
  })

  it('reaps a legacy_flat orphan through the legacy predicate and deletes the flat bytes', async () => {
    const resourceId = randomUUID()
    const legacyPath = resolveBlobPath(blobRoot, resourceId)
    // A legacy_flat blob's manifest key IS the dash-stripped resource id (the
    // blob_key_valid CHECK: `blob_key = replace(resource_id,'-','')`); legacy
    // candidates carry no committed digest (content_sha256 IS NULL).
    const legacyBlobKey = resourceId.replace(/-/g, '')
    await writeFile(legacyPath, randomBytes(48), { mode: 0o600 })
    await manifests.recordStaged({
      blobKey: legacyBlobKey,
      requestId: randomUUID(),
      resourceId,
      candidateKind: 'legacy_flat',
      contentSha256: null,
      bytes: 48,
    })
    await ageCandidate(legacyBlobKey)

    expect(existsSync(legacyPath)).toBe(true)
    const recorder = metricsStub()
    const result = await reconcileExpiredBlobs(manifests, blobs, recorder.metrics, {
      olderThanMs: 1000,
      limit: 16,
    })

    expect(result.deleted).toBe(1)
    expect(result.failures).toBe(0)
    // Read the recording stub too: the reconciler's failure metric must stay
    // untouched, not just its returned tally.
    expect(recorder.failures).toBe(0)
    // Business-truth: the flat legacy bytes are gone and the manifest row is gone.
    expect(existsSync(legacyPath)).toBe(false)
    expect(await manifestExists(legacyBlobKey)).toBe(false)
    expect(await manifestState(legacyBlobKey)).toBeNull()
  })
})