Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
8.2 KB
Raw Blame
/**
 * LLM catalog discovery sync cron (spec 09 Fase 4).
 *
 * Periodically runs `syncDiscoveredModels()` — the SAME non-destructive
 * reconciliation the on-demand `POST /admin/llm-models/discovery/sync` endpoint
 * drives. No HTTP hop, no new auth: the tick calls the service directly. New
 * models land as `enabled=false, source='discovery'` (inert until an operator
 * enables them); vanished models are only ever flagged `stale=true` (never
 * deleted, never disabled), and only when the §4.5 sanity guards agree the live
 * catalog is trustworthy.
 *
 * Cross-replica dedup: control-api runs multiple replicas, and 24h ticks across
 * them would otherwise pile up redundant runs. Each tick wraps its work in a
 * SESSION-scoped `pg_try_advisory_lock(hashtext('llm-catalog-sync-cron-v1'))`
 * on a dedicated client. If another replica already holds it, this tick is a
 * no-op. A session lock (unlike the xact lock inside syncDiscoveredModels) is
 * NOT auto-released on COMMIT, so it MUST be explicitly unlocked — a leaked lock
 * would wedge every future tick. The `finally` guarantees unlock + release.
 *
 * The service's own `pg_advisory_xact_lock` still runs and still serializes a
 * concurrent on-demand sync against this cron; the session lock here is an
 * ADDITIONAL guard that avoids queuing redundant cron ticks behind it.
 *
 * Scheduling follows workflowScheduleWorkerCron.ts: the first tick runs shortly
 * after start (random jitter up to LLM_CATALOG_SYNC_FIRST_RUN_JITTER_MS), then
 * every `intervalMs`. Both timers are unref()'d, start is not awaited by boot,
 * and errors are logged but NEVER thrown.
 */
import { pool } from '../db.js'
import { rootLogger } from '../observability/logger.js'
import type { CatalogSyncResult } from './llmCatalogSync.js'

const log = rootLogger.child({ service: 'llm_catalog_sync_cron' })

// Session-scoped advisory lock key. `-cron-v1` distinguishes it from the
// service's own xact lock (`llm-catalog-sync-v1`) — they are different locks
// with different scopes and must not collide.
const CRON_LOCK_KEY_SQL = "hashtext('llm-catalog-sync-cron-v1')"

/**
 * Minimal client shape for the advisory-lock session. pg's PoolClient satisfies
 * it. `release` accepts pg's optional destroy argument: passing a truthy value
 * (an Error) DESTROYS the connection instead of returning it to the pool — the
 * only way to guarantee a SESSION advisory lock is freed when unlock itself fails.
 */
type LockClient = {
  query: (text: string, values?: unknown[]) => Promise<{ rows: unknown[]; rowCount: number | null }>
  release: (destroy?: Error | boolean) => void
}

/**
 * Injectable dependencies. `connector` defaults to production wiring; `sync` is
 * REQUIRED because the sync now needs a ConfigMap materializer (#654) and the
 * cron has no gateway of its own — `main.ts` owns that wiring and passes a
 * closure. A default here could only be a call the sync would reject.
 */
export interface LlmCatalogSyncCronDeps {
  connector?: { connect: () => Promise<LockClient> }
  sync: () => Promise<CatalogSyncResult>
}

/** Outcome of one tick — returned for tests/observability, never thrown. */
export interface LlmCatalogSyncTickResult {
  skippedLock: boolean
  ran: boolean
  errored: boolean
}

/**
 * Run one cron tick: acquire the session advisory lock, run the sync, release.
 * Never throws — a failure is logged and reported in the result. The lock is
 * released in `finally` whether the tick ran, skipped, or failed.
 */
export async function runLlmCatalogSyncTick(
  deps: LlmCatalogSyncCronDeps
): Promise<LlmCatalogSyncTickResult> {
  const connector = deps.connector ?? pool
  const sync = deps.sync
  let lockClient: LockClient | undefined
  let locked = false
  try {
    lockClient = await connector.connect()
    const lockRes = await lockClient.query(
      `SELECT pg_try_advisory_lock(${CRON_LOCK_KEY_SQL}) AS acquired`
    )
    locked = (lockRes.rows[0] as { acquired?: boolean } | undefined)?.acquired === true
    if (!locked) {
      log.debug(
        { event: 'llm_catalog_sync_cron_skipped_lock' },
        'llm catalog sync tick skipped: advisory lock held by another replica'
      )
      return { skippedLock: true, ran: false, errored: false }
    }

    const result = await sync()
    log.info(
      {
        event: 'llm_catalog_sync_cron_tick',
        source: result.source,
        added: result.added,
        updated: result.updated,
        staled: result.staled,
      },
      `llm catalog sync tick complete (source=${result.source}, added=${result.added}, updated=${result.updated}, staled=${result.staled})`
    )
    return { skippedLock: false, ran: true, errored: false }
  } catch (err) {
    log.error(
      {
        event: 'llm_catalog_sync_cron_error',
        err: err instanceof Error ? err.message : String(err),
      },
      'llm catalog sync tick failed'
    )
    return { skippedLock: false, ran: false, errored: true }
  } finally {
    if (lockClient) {
      // SESSION lock → explicit unlock is mandatory; only unlock what we acquired.
      let destroyed = false
      if (locked) {
        try {
          await lockClient.query(`SELECT pg_advisory_unlock(${CRON_LOCK_KEY_SQL})`)
        } catch (unlockErr) {
          // A SESSION lock is NOT freed by returning the connection to the pool —
          // only by an explicit unlock or by CLOSING the session. Since the unlock
          // failed, DESTROY the connection (release(err) ends the session), which
          // guarantees the lock is released. A plain release() here would hand a
          // live connection still holding the lock back to the pool and wedge every
          // future tick — exactly the failure the module header promises to avoid.
          const asError = unlockErr instanceof Error ? unlockErr : new Error(String(unlockErr))
          log.warn(
            {
              event: 'llm_catalog_sync_cron_unlock_failed',
              err: asError.message,
            },
            'pg_advisory_unlock failed; destroying the connection to release the session lock'
          )
          lockClient.release(asError)
          destroyed = true
        }
      }
      // Only return the connection to the pool when we did NOT destroy it above
      // (destroying already released it — a second release would double-release).
      if (!destroyed) lockClient.release()
    }
  }
}

let intervalHandle: ReturnType<typeof setInterval> | null = null
let firstRunHandle: ReturnType<typeof setTimeout> | null = null

/**
 * Upper bound of the random delay before the first tick after start. Keeps the
 * replicas of one rollout from all contending for the advisory lock at the same
 * instant; the lock already makes a collision a harmless no-op.
 */
export const LLM_CATALOG_SYNC_FIRST_RUN_JITTER_MS = 5_000

export function startLlmCatalogSyncCron(deps: LlmCatalogSyncCronDeps, intervalMs: number): void {
  if (intervalHandle || firstRunHandle) return

  const run = (): void => {
    // runLlmCatalogSyncTick never rejects, but keep a defensive catch so an
    // unexpected throw can never crash the timer.
    void runLlmCatalogSyncTick(deps).catch(err => {
      log.error(
        {
          event: 'llm_catalog_sync_cron_unhandled',
          err: err instanceof Error ? err.message : String(err),
        },
        'unhandled error in llm catalog sync cron'
      )
    })
  }

  // First tick shortly after start instead of one full interval later: rows
  // with no image-input evidence refuse images until a sync runs, and every
  // restart would otherwise push that first sync out by another interval.
  // Not awaited — boot never waits on models.dev.
  const firstDelay = Math.floor(Math.random() * LLM_CATALOG_SYNC_FIRST_RUN_JITTER_MS)
  firstRunHandle = setTimeout(() => {
    firstRunHandle = null
    run()
    intervalHandle = setInterval(run, intervalMs)
    intervalHandle.unref()
  }, firstDelay)
  firstRunHandle.unref()
  log.info(
    { event: 'llm_catalog_sync_cron_started', intervalMs, firstRunInMs: firstDelay },
    'llm catalog sync cron started'
  )
}

export function stopLlmCatalogSyncCron(): void {
  if (firstRunHandle) {
    clearTimeout(firstRunHandle)
    firstRunHandle = null
  }
  if (intervalHandle) {
    clearInterval(intervalHandle)
    intervalHandle = null
  }
}