Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
20.4 KB
Raw Blame
import { Client } from "pg";

/**
 * Readiness probe for the permission store (issue #775).
 *
 * The naive `pool.connect()` + `SELECT 1` check cannot detect a rotated
 * database credential on a long-lived pod: the pool reuses idle clients that
 * authenticated BEFORE the rotation, and the kubelet readiness cadence (~5s)
 * keeps at least one such client alive forever (pg idle timeout 10s), so
 * readiness stays green while every NEW connection fails auth and every
 * request 503s. This probe closes that hole with two layers:
 *
 * 1. Fast pool ping (every call) — preserves the original check; detects a
 *    down/unreachable store within one probe period.
 * 2. Amortized fresh-connection probe (at most once per `intervalMs`) — opens
 *    a brand-new client (NEVER the pool) so a rotated credential fails to
 *    authenticate, and verifies real coherence with `has_table_privilege`, so
 *    "password correct but role grants missing" also fails loud. The writer's
 *    grants come from migration 0048_gfs_permission_store, the reader's from
 *    0072_gfs_reader_database_role, and 0074_gfs_runtime_role_exact_contract
 *    rebuilds both envelopes.
 *
 * FAIL CLOSED: only SUCCESS is cached. While failing, every readiness call
 * retries the fresh connection (~1 new connection per probe period during an
 * outage — exactly when it matters). Error messages never include the DSN.
 */

/** Minimal pool surface (structurally satisfied by pg.Pool; injectable in tests). */
export interface PingPool {
  connect(): Promise<{
    query(sql: string): Promise<unknown>;
    /** Passing an error DESTROYS the client instead of returning it to the pool. */
    release(err?: Error): void;
  }>;
}
/** Minimal ephemeral-client surface (structurally satisfied by pg.Client). */
export interface ProbeClient {
  connect(): Promise<void>;
  query(sql: string): Promise<{ rows: Array<Record<string, unknown>> }>;
  end(): Promise<void>;
}

export interface StoreProbeOptions {
  /** Shared pool used by the serving path — fast liveness ping only. */
  pool: PingPool;
  /** Same DSN the pool uses; the fresh probe dials it with a NEW client. */
  connectionString: string;
  /** Amortization window for the fresh-connection probe (e.g. 60_000). */
  intervalMs: number;
  /** Selects the exact least-privilege readiness contract; never inferred. */
  storageRole: "writer" | "reader";
  /**
   * Hard bound on each fresh-probe step (connect AND coherence query).
   * Without it, a black-hole partition (unreachable but not refusing) leaves
   * connect() pending for the OS TCP timeout (~minutes) while the kubelet
   * fires a new probe every few seconds — an unbounded pile-up of dangling
   * connection attempts. Default 5_000.
   */
  connectTimeoutMs?: number;
  /** Injectable for tests; defaults to a real pg.Client. */
  clientFactory?: (connectionString: string) => ProbeClient;
  /** Injectable clock for tests; defaults to Date.now. */
  now?: () => number;
}

function errMsg(err: unknown): string {
  return err instanceof Error ? err.message : String(err);
}

/** Bounded await — the probe must fail loud and fast, never dangle. */
function withTimeout<T>(promise: Promise<T>, ms: number, what: string): Promise<T> {
  let timer: ReturnType<typeof setTimeout> | undefined;
  const deadline = new Promise<never>((_, reject) => {
    timer = setTimeout(
      () => reject(new Error(`${what} timed out after ${ms}ms (permission store unreachable?)`)),
      ms
    );
    timer.unref?.();
  });
  return Promise.race([promise, deadline]).finally(() => clearTimeout(timer)) as Promise<T>;
}

// Coherence canary: both halves of the role's least-privilege contract. A
// broken SELECT on gfs_resources breaks every authorization lookup; a broken
// INSERT on gfs_audit breaks every request too (audit-write failures propagate
// — no un-audited op is served), so either one must flip readiness instead of
// surfacing as chronic per-request 503s.
const BASE_COHERENCE_SQL =
  "SELECT has_table_privilege(current_user, 'gfs_resources', 'SELECT') AS can_read, " +
  "has_table_privilege(current_user, 'gfs_audit', 'INSERT') AS can_audit, " +
  "has_column_privilege(current_user, 'gfs_resources', 'blob_key', 'SELECT') AS can_read_blob_key, " +
  "has_column_privilege(current_user, 'gfs_resources', 'content_sha256', 'SELECT') AS can_read_digest, " +
  "EXISTS (SELECT 1 FROM (SELECT blob_key, content_sha256 FROM gfs_resources WHERE false) AS resource_probe) = false AS resource_schema_ready, " +
  "(SELECT count(*) = 5 " +
  "AND bool_and(attname = 'record_type' OR atttypid = 'text'::regtype) " +
  "AND bool_and(attname <> 'record_type' OR attnotnull) FROM pg_attribute " +
  "WHERE attrelid = 'gfs_audit'::regclass AND attnum > 0 AND NOT attisdropped " +
  "AND attname = ANY (ARRAY['record_type', 'matched_subject', 'authorization_source', 'cached_authorization_source', 'mutation_outcome'])) AS audit_decision_evidence_schema_ready, " +
  "(SELECT count(*) = 5 FROM pg_constraint WHERE conrelid = 'gfs_audit'::regclass " +
  "AND conname = ANY (ARRAY['gfs_audit_record_type_valid', 'gfs_audit_authorization_source_valid', 'gfs_audit_cached_authorization_source_valid', 'gfs_audit_mutation_outcome_valid', 'gfs_audit_record_type_fields_valid']) " +
  "AND contype = 'c' AND convalidated) AS audit_decision_evidence_constraints_ready, " +
  "(SELECT count(*) = 3 " +
  "AND bool_and((attname = 'actor_on_behalf_of' AND atttypid = 'text'::regtype AND NOT attnotnull) " +
  "OR (attname = 'desktop_user_id' AND atttypid = 'uuid'::regtype AND NOT attnotnull) " +
  "OR (attname = 'authority_source' AND atttypid = 'text'::regtype AND NOT attnotnull)) " +
  "FROM pg_attribute WHERE attrelid = 'gfs_audit'::regclass AND attnum > 0 AND NOT attisdropped " +
  "AND attname = ANY (ARRAY['actor_on_behalf_of', 'desktop_user_id', 'authority_source'])) AS audit_actor_correlation_columns_ready, " +
  "EXISTS (SELECT 1 FROM pg_constraint WHERE conrelid = 'gfs_audit'::regclass " +
  "AND conname = 'gfs_audit_actor_correlation_valid' AND contype = 'c' AND convalidated " +
  "AND regexp_replace(pg_get_constraintdef(oid), '\\s+', '', 'g') LIKE '%desktop_user_idISNULL%' " +
  "AND regexp_replace(pg_get_constraintdef(oid), '\\s+', '', 'g') LIKE '%authority_sourceISNULL%' " +
  "AND regexp_replace(pg_get_constraintdef(oid), '\\s+', '', 'g') LIKE '%authority_source=''user-session''%' " +
  "AND regexp_replace(pg_get_constraintdef(oid), '\\s+', '', 'g') LIKE '%authority_source=''linked-admin''%' " +
  "AND regexp_replace(pg_get_constraintdef(oid), '\\s+', '', 'g') LIKE '%actor_on_behalf_ofISNULL%' " +
  "AND regexp_replace(pg_get_constraintdef(oid), '\\s+', '', 'g') LIKE '%actor_on_behalf_ofISNOTNULL%') AS audit_actor_correlation_constraint_ready";

const AUTHORITY_PROJECTION_SQL =
  ", (SELECT count(*) = 3 AND bool_and((attname = 'id' AND atttypid = 'uuid'::regtype) OR " +
  "(attname = 'lifecycle_state' AND atttypid = 'text'::regtype AND attnotnull) OR " +
  "(attname = 'lifecycle_version' AND atttypid = 'int8'::regtype AND attnotnull)) " +
  "FROM pg_attribute WHERE attrelid = 'users'::regclass AND attnum > 0 AND NOT attisdropped " +
  "AND attname = ANY (ARRAY['id','lifecycle_state','lifecycle_version'])) AS authority_users_columns_ready" +
  ", (SELECT count(*) = 3 AND bool_and((attname = 'id' AND atttypid = 'uuid'::regtype) OR " +
  "(attname = 'status' AND atttypid = 'text'::regtype) OR " +
  "(attname = 'session_version' AND atttypid = 'int4'::regtype)) " +
  "FROM pg_attribute WHERE attrelid = 'control_admin_users'::regclass AND attnum > 0 AND NOT attisdropped " +
  "AND attname = ANY (ARRAY['id','status','session_version'])) AS authority_admin_columns_ready" +
  ", (SELECT count(*) = 1 " +
  "AND EXISTS (SELECT 1 FROM pg_attrdef defs JOIN pg_attribute attrs " +
  "ON attrs.attrelid = defs.adrelid AND attrs.attnum = defs.adnum " +
  "WHERE defs.adrelid = 'control_admin_users'::regclass " +
  "AND attrs.attname = 'session_version' " +
  "AND attrs.attnotnull " +
  "AND pg_get_expr(defs.adbin, defs.adrelid) IN ('1', '1::integer')) " +
  "AND NOT EXISTS (SELECT 1 FROM control_admin_users WHERE session_version < 1) " +
  "AND EXISTS (SELECT 1 FROM pg_constraint " +
  "WHERE conrelid = 'control_admin_users'::regclass " +
  "AND conname = 'control_admin_session_version_positive' " +
  "AND contype = 'c' AND convalidated) " +
  "FROM pg_attribute WHERE attrelid = 'control_admin_users'::regclass " +
  "AND attname = 'session_version' AND attnum > 0 AND NOT attisdropped) " +
  'AS authority_admin_session_epoch_ready' +
  ", (SELECT count(*) = 7 AND bool_and((attname = 'id' AND atttypid = 'uuid'::regtype) OR " +
  "(attname = 'lineage_id' AND atttypid = 'uuid'::regtype) OR " +
  "(attname = 'generation' AND atttypid = 'int4'::regtype) OR " +
  "(attname = 'user_id' AND atttypid = 'uuid'::regtype) OR " +
  "(attname = 'control_admin_id' AND atttypid = 'uuid'::regtype) OR " +
  "(attname = 'state' AND atttypid = 'text'::regtype) OR " +
  "(attname = 'source' AND atttypid = 'text'::regtype)) " +
  "FROM pg_attribute WHERE attrelid = 'gfs_desktop_operator_links'::regclass AND attnum > 0 AND NOT attisdropped " +
  "AND attname = ANY (ARRAY['id','lineage_id','generation','user_id','control_admin_id','state','source'])) AS authority_links_columns_ready" +
  ", (CASE WHEN to_regclass('public.users') IS NULL THEN false ELSE " +
  "has_column_privilege(current_user, 'public.users', 'id', 'SELECT') " +
  "AND has_column_privilege(current_user, 'public.users', 'lifecycle_state', 'SELECT') " +
  "AND has_column_privilege(current_user, 'public.users', 'lifecycle_version', 'SELECT') END) AS authority_users_privileges_ready" +
  ", (CASE WHEN to_regclass('public.control_admin_users') IS NULL THEN false ELSE " +
  "has_column_privilege(current_user, 'public.control_admin_users', 'id', 'SELECT') " +
  "AND has_column_privilege(current_user, 'public.control_admin_users', 'status', 'SELECT') " +
  "AND has_column_privilege(current_user, 'public.control_admin_users', 'session_version', 'SELECT') END) AS authority_admin_privileges_ready" +
  ", (CASE WHEN to_regclass('public.gfs_desktop_operator_links') IS NULL THEN false ELSE " +
  "has_column_privilege(current_user, 'public.gfs_desktop_operator_links', 'id', 'SELECT') " +
  "AND has_column_privilege(current_user, 'public.gfs_desktop_operator_links', 'lineage_id', 'SELECT') " +
  "AND has_column_privilege(current_user, 'public.gfs_desktop_operator_links', 'generation', 'SELECT') " +
  "AND has_column_privilege(current_user, 'public.gfs_desktop_operator_links', 'user_id', 'SELECT') " +
  "AND has_column_privilege(current_user, 'public.gfs_desktop_operator_links', 'control_admin_id', 'SELECT') " +
  "AND has_column_privilege(current_user, 'public.gfs_desktop_operator_links', 'state', 'SELECT') " +
  "AND has_column_privilege(current_user, 'public.gfs_desktop_operator_links', 'source', 'SELECT') END) AS authority_links_privileges_ready";

// The reader is deliberately read-only except for append-only audit records.
// Positive checks alone are insufficient: a stale/manual grant would keep the
// role Ready while silently expanding its blast radius. These checks use the
// effective current_user privileges, so inherited or PUBLIC grants fail too.
const READER_COHERENCE_SQL =
  BASE_COHERENCE_SQL +
  AUTHORITY_PROJECTION_SQL +
  ", has_table_privilege(current_user, 'gfs_resources', 'INSERT, UPDATE, DELETE, TRUNCATE') AS can_mutate_resources, " +
  "has_table_privilege(current_user, 'gfs_grants', 'INSERT, UPDATE, DELETE, TRUNCATE') AS can_mutate_grants, " +
  "has_table_privilege(current_user, 'gfs_shares', 'INSERT, UPDATE, DELETE, TRUNCATE') AS can_mutate_shares, " +
  "has_table_privilege(current_user, 'gfs_blob_manifests', 'INSERT, UPDATE, DELETE, TRUNCATE') AS can_mutate_manifests, " +
  "has_table_privilege(current_user, 'gfs_audit', 'UPDATE, DELETE, TRUNCATE') AS can_mutate_audit, " +
  "has_sequence_privilege(current_user, 'gfs_audit_sequence_no_seq', 'UPDATE') AS can_update_audit_sequence";

const WRITER_COHERENCE_SQL =
  BASE_COHERENCE_SQL +
  AUTHORITY_PROJECTION_SQL +
  ", has_table_privilege(current_user, 'gfs_blob_manifests', 'SELECT') AS can_read_manifests, " +
  "has_table_privilege(current_user, 'gfs_blob_manifests', 'INSERT') AS can_insert_manifests, " +
  "has_table_privilege(current_user, 'gfs_blob_manifests', 'UPDATE') AS can_update_manifests, " +
  "has_table_privilege(current_user, 'gfs_blob_manifests', 'DELETE') AS can_delete_manifests, " +
  "has_table_privilege(current_user, 'gfs_resources', 'INSERT') AS can_insert_resources, " +
  "has_table_privilege(current_user, 'gfs_resources', 'UPDATE') AS can_update_resources, " +
  "EXISTS (SELECT 1 FROM (SELECT blob_key, candidate_kind, state, content_sha256, bytes FROM gfs_blob_manifests WHERE false) AS manifest_probe) = false AS manifest_schema_ready";

export function createPermissionStoreProbe(opts: StoreProbeOptions): () => Promise<void> {
  const now = opts.now ?? Date.now;
  const connectTimeoutMs = opts.connectTimeoutMs ?? 5_000;
  // Each role gets its base grants from its own migration; 0074 later revokes
  // every table and column privilege and rebuilds both envelopes exactly.
  const roleGrantsMigration =
    opts.storageRole === "writer" ? "0048_gfs_permission_store" : "0072_gfs_reader_database_role";
  const roleGrantsHint =
    `control-api migration ${roleGrantsMigration} not applied, ` +
    "or 0074_gfs_runtime_role_exact_contract grants drifted?";
  // @types/pg resolves Client.connect() to Promise<Client>; the probe contract
  // wants Promise<void> — awaited and discarded (same adaptation as the LISTEN
  // client in index.ts). connectionTimeoutMillis is belt-and-suspenders under
  // the probe-level withTimeout bound (which also covers the query).
  const clientFactory =
    opts.clientFactory ??
    ((connectionString: string): ProbeClient => {
      const client = new Client({ connectionString, connectionTimeoutMillis: connectTimeoutMs });
      return {
        connect: async () => {
          await client.connect();
        },
        query: (sql: string) => client.query(sql),
        end: () => client.end(),
      };
    });
  let lastFreshOkAt: number | null = null;

  return async (): Promise<void> => {
    // Layer 1 — fast pool ping (original check, always runs), bounded like
    // Layer 2. The acquire needs orphan-handling that the ephemeral client
    // does not: when the deadline wins the race, the underlying
    // pool.connect() keeps running, and if it EVENTUALLY succeeds (a
    // partition clearing) the checked-out client must be returned or the
    // pool slot leaks — sustained probes would starve the serving path's own
    // pool. Ordering is safe: the race rejection's await continuation (which
    // sets the flag) is queued before any later acquire resolution.
    let acquireTimedOut = false;
    const acquire = opts.pool.connect();
    acquire.then(
      (client) => {
        if (acquireTimedOut) client.release();
      },
      () => undefined // late rejection: nothing was checked out
    );
    let pooled: Awaited<ReturnType<PingPool["connect"]>>;
    try {
      pooled = await withTimeout(acquire, connectTimeoutMs, "permission store pool ping");
    } catch (err) {
      acquireTimedOut = true;
      throw err;
    }
    let pingFailure: Error | undefined;
    try {
      await withTimeout(pooled.query("SELECT 1"), connectTimeoutMs, "permission store pool ping");
    } catch (err) {
      // DESTROY (release-with-error) a client whose ping hung or failed —
      // returning a connection with an in-flight query to the pool would
      // poison later checkouts.
      pingFailure = err instanceof Error ? err : new Error(String(err));
      throw err;
    } finally {
      if (pingFailure) {
        pooled.release(pingFailure);
      } else {
        pooled.release();
      }
    }

    // Layer 2 — amortized fresh-connection credential + coherence probe.
    if (lastFreshOkAt !== null && now() - lastFreshOkAt < opts.intervalMs) {
      return;
    }
    const client = clientFactory(opts.connectionString);
    try {
      // A rotated credential fails HERE with the real auth error — the pool
      // would have masked it by reusing a pre-rotation connection. Both steps
      // are hard-bounded so a black-hole partition fails this probe cycle
      // instead of dangling until the OS TCP timeout.
      await withTimeout(client.connect(), connectTimeoutMs, "permission store connect");
      const result = await withTimeout(
        client.query(opts.storageRole === "writer" ? WRITER_COHERENCE_SQL : READER_COHERENCE_SQL),
        connectTimeoutMs,
        "permission store coherence query"
      );
      const row = result.rows[0];
      if (row?.can_read !== true) {
        throw new Error(
          "permission store coherence check failed: role lacks SELECT on gfs_resources " +
            `(${roleGrantsHint})`
        );
      }
      if (row?.can_audit !== true) {
        throw new Error(
          "permission store coherence check failed: role lacks INSERT on gfs_audit " +
            `(audit-write failures would 503 every request; ${roleGrantsHint})`
        );
      }
      if (
        row?.can_read_blob_key !== true ||
        row?.can_read_digest !== true ||
        row?.resource_schema_ready !== true
      ) {
        throw new Error(
          "permission store coherence check failed: immutable GFS blob schema/reader privileges " +
            "are missing (control-api migration 0071_gfs_immutable_blob_generations not applied, " +
            "or 0074_gfs_runtime_role_exact_contract grants drifted?)"
        );
      }
      if (
        row?.audit_decision_evidence_schema_ready !== true ||
        row?.audit_decision_evidence_constraints_ready !== true
      ) {
        throw new Error(
          "permission store coherence check failed: GFS audit decision-evidence schema is missing " +
            "(control-api migration 0073_gfs_audit_decision_evidence not applied?)"
        );
      }
      if (
        row?.audit_actor_correlation_columns_ready !== true ||
        row?.audit_actor_correlation_constraint_ready !== true
      ) {
        throw new Error(
          "permission store coherence check failed: GFS audit actor-correlation schema is missing " +
            "(control-api migration 0092_gfs_audit_actor_correlation not applied?)"
        );
      }
      if (
        row?.authority_users_columns_ready !== true ||
        row?.authority_admin_columns_ready !== true ||
        row?.authority_admin_session_epoch_ready !== true ||
        row?.authority_links_columns_ready !== true ||
        row?.authority_users_privileges_ready !== true ||
        row?.authority_admin_privileges_ready !== true ||
        row?.authority_links_privileges_ready !== true
      ) {
        throw new Error(
          "permission store coherence check failed: GFS lifecycle authority projection is missing " +
            "(control-api migrations 0095_gfs_lifecycle_authority_projection/0096_control_admin_session_version_default not applied?)"
        );
      }
      if (
        opts.storageRole === "reader" &&
        (row?.can_mutate_resources !== false ||
          row?.can_mutate_grants !== false ||
          row?.can_mutate_shares !== false ||
          row?.can_mutate_manifests !== false ||
          row?.can_mutate_audit !== false ||
          row?.can_update_audit_sequence !== false)
      ) {
        throw new Error(
          "permission store coherence check failed: reader has forbidden GFS mutation privileges " +
            "(control-api migration 0074_gfs_runtime_role_exact_contract grants drifted?)"
        );
      }
      if (
        opts.storageRole === "writer" &&
        (row?.can_read_manifests !== true ||
          row?.manifest_schema_ready !== true ||
          row?.can_insert_manifests !== true ||
          row?.can_update_manifests !== true ||
          row?.can_delete_manifests !== true ||
          row?.can_insert_resources !== true ||
          row?.can_update_resources !== true)
      ) {
        throw new Error(
          "permission store coherence check failed: writer lacks GFS manifest/resource mutation " +
            "privileges (control-api migration 0071_gfs_immutable_blob_generations not applied, " +
            "or 0074_gfs_runtime_role_exact_contract grants drifted?)"
        );
      }
      lastFreshOkAt = now(); // cache SUCCESS only — failures always retry
    } finally {
      // Best-effort close of the ephemeral client; a close error must not mask
      // the probe verdict, but it is logged loudly (never swallowed silently).
      await client.end().catch((err) => {
        console.error(`[gfsc] store-probe client close error: ${errMsg(err)}`);
      });
    }
  };
}