Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
20.6 KB
Raw Blame
import { afterAll, beforeAll, describe, expect, it } from 'vitest'
import { randomBytes, randomUUID } from 'node:crypto'
import { Pool } from 'pg'
import type { PluginWorkloadSdkFamily } from '../src/services/pluginWorkloadSdkDb.js'
import type { McpHostAccessClaims } from '../src/utils/auth/mcpHostJwtToken.js'
import { endPoolAndWaitForClients } from './helpers/realPostgresTeardown.js'

/**
 * Stepless / SDK-only quota regression on real PostgreSQL (issue #348,
 * plan D4) — THE ACCEPTANCE-DEFINING RED-FIRST TEST.
 *
 * Test 1 asserts the POST-change target behavior and MUST FAIL against
 * pre-change code (RED): today a stepless recipe (zero `workflow_runs` rows)
 * books every per-run quota unit under the immortal epoch-sentinel bucket
 * (`resolveQuotaPeriodStart` → `PLUGIN_WORKLOAD_SDK_EAGER_QUOTA_PERIOD =
 * new Date(0)`), so a per-run cap becomes a lifetime cap that nothing ever
 * resets. After Phase 1 the per-run leg is inert and every call is allowed.
 * Do NOT weaken these assertions to pass early.
 *
 * POST-PHASE-1 NOTE (applied, audit finding 1): Phase 1 DELETED `consumeQuota`
 * outright (plan decision §3.2 — delete, don't stub; the RED runs were
 * captured against the pre-change `consumeQuota` API before the deletion).
 * Tests 1 and 3 exercise the REAL enforcement entry points —
 * `authorizePromptBridge` and (mirror) `authorizeClientNotification` in
 * `src/services/pluginWorkloadSdkAuthorizer.ts` — against the real database.
 * Pointing them at `checkRateLimit` alone would be near-vacuous: that function
 * is the per-MINUTE gate and never consulted the per-RUN cap or
 * `plugin_workload_sdk_quota_counters` even pre-change, so a future re-wire of
 * the per-run leg inside the AUTHORIZER would not trip it. Going through the
 * authorizer means ANY quota consultation reachable from the authorization
 * pipeline (present or reintroduced) is under test. The assertions:
 *   (a) the authorizer never denies a stepless recipe past its declared
 *       per-run cap (the deprecated cap is inert), and
 *   (b) `plugin_workload_sdk_quota_counters` receives NO writes on that path
 *       (the epoch-sentinel bucket is never touched) — counted before/after.
 * Test 2 stays on `checkRateLimit` deliberately: the per-minute window/reset
 * semantics are exactly that function's contract.
 *
 * Real database, real `plugin_workload_sdk_quota_counters` and
 * `plugin_workload_sdk_invocations` tables — NO db mock. The service module's
 * pool (`src/db.ts` module-level `pool`, built from
 * `CONTROL_API_PG_CONNECTION_STRING` at import time) is pointed at a fresh
 * throwaway database by setting that env var BEFORE the src modules are
 * dynamically imported in beforeAll; nothing from `src/` is value-imported
 * statically in this file for exactly that reason.
 */

const adminUrl = process.env.CONTROL_API_REAL_PG_ADMIN_URL
// Repo convention (matches the other *.realPostgres.integration.test.ts suites):
// gate on the DB env so the DB-less unit lane stays green. This suite RUNS
// non-skipped in CI via the Postgres-backed lane in
// .github/workflows/ci-public.yml (which sets CONTROL_API_REAL_PG_ADMIN_URL
// against a Postgres 16 service and runs the realPostgres suites), and was
// proven live on minikube (issue #348, plan D4). Local DB-less runs skip per
// owner decision.
const describeRealPostgres = adminUrl ? describe : describe.skip

const database = `control_api_sdk_stepless_${randomBytes(6).toString('hex')}`

function databaseUrl(baseUrl: string, databaseName: string): string {
  const url = new URL(baseUrl)
  url.pathname = `/${databaseName}`
  return url.toString()
}

// Redirect the module-level pool of src/db.ts to the throwaway database.
// This runs at import time, strictly before the dynamic src imports below.
// Guarded on adminUrl so the DB-less (skipped) lane never builds a URL from undefined.
const previousPgConnectionString = process.env.CONTROL_API_PG_CONNECTION_STRING
if (adminUrl) {
  process.env.CONTROL_API_PG_CONNECTION_STRING = databaseUrl(adminUrl, database)
}

const NS = 'sandbox-recipes'
const CALLER_REF = 'api'

/** Claims fixture mirrored from test/services.pluginWorkloadSdkAuthorizer.test.ts. */
function claimsFor(recipeName: string): McpHostAccessClaims {
  return {
    sub: `${NS}/${recipeName}`,
    recipeNamespace: NS,
    recipeName,
    hostRefs: [`${NS}/${recipeName}`],
    scope: 'workflow:approval:request',
    workflowControlScopes: ['plugin-workload-sdk'] as McpHostAccessClaims['workflowControlScopes'],
    iss: 'control-api',
    aud: 'workflow-approvals',
    jti: 'test-jti',
    exp: Math.floor(Date.now() / 1000) + 300,
  }
}

describeRealPostgres(
  'Plugin Workload SDK stepless quota enforcement on real PostgreSQL (issue #348)',
  () => {
    let adminPool: Pool
    let db: typeof import('../src/db.js')
    let quotaTracker: typeof import('../src/services/pluginWorkloadSdkQuotaTracker.js')
    let authorizer: typeof import('../src/services/pluginWorkloadSdkAuthorizer.js')

    beforeAll(async () => {
      adminPool = new Pool({ connectionString: adminUrl })
      await adminPool.query(`CREATE DATABASE "${database.replace(/"/g, '""')}"`)
      // Import AFTER the database exists and AFTER the env override above so
      // the module-level pool binds to the fresh database.
      db = await import('../src/db.js')
      await db.initDb()
      quotaTracker = await import('../src/services/pluginWorkloadSdkQuotaTracker.js')
      authorizer = await import('../src/services/pluginWorkloadSdkAuthorizer.js')
    })

    afterAll(async () => {
      try {
        await endPoolAndWaitForClients(db?.pool)
        await endPoolAndWaitForClients(db?.rateLimitPool)
        if (previousPgConnectionString === undefined) {
          delete process.env.CONTROL_API_PG_CONNECTION_STRING
        } else {
          process.env.CONTROL_API_PG_CONNECTION_STRING = previousPgConnectionString
        }
        if (adminPool) {
          await adminPool.query(
            `SELECT pg_terminate_backend(pid)
           FROM pg_stat_activity
          WHERE datname = $1
            AND pid <> pg_backend_pid()`,
            [database]
          )
          await adminPool.query(`DROP DATABASE IF EXISTS "${database.replace(/"/g, '""')}"`)
        }
      } finally {
        await adminPool?.end()
      }
    })

    /** Fail-loud stepless precondition: the recipe has ZERO workflow_runs rows. */
    async function assertStepless(recipeName: string): Promise<void> {
      const runs = await db.pool.query<{ n: number }>(
        `SELECT count(*)::int AS n FROM workflow_runs
        WHERE recipe_namespace = $1 AND recipe_name = $2`,
        [NS, recipeName]
      )
      expect(runs.rows[0]?.n, `stepless precondition for ${recipeName}`).toBe(0)
    }

    /** Rows currently booked in the (deprecated) per-run counter bucket table. */
    async function quotaCounterRowCount(recipeName: string): Promise<number> {
      const result = await db.pool.query<{ n: number }>(
        `SELECT count(*)::int AS n FROM plugin_workload_sdk_quota_counters
        WHERE recipe_namespace = $1 AND recipe_name = $2`,
        [NS, recipeName]
      )
      return result.rows[0]?.n ?? 0
    }

    /**
     * Seed an operator-reviewed, ACTIVE promptBridge grant exactly as the
     * authorizer requires it (policy_revision >= 1, first ordered target ==
     * default_target_ref, durable review provenance). The INSERT mirrors the
     * grant seeding of pluginWorkloadSdkPolicyReview.realPostgres.integration
     * .test.ts, extended with quota_limits so the deprecated per-run cap is on
     * the row the authorizer actually loads via findGrant.
     */
    async function seedPromptBridgeGrant(
      recipeName: string,
      quotaLimits: Record<string, number>
    ): Promise<void> {
      await db.pool.query(
        `INSERT INTO plugin_workload_sdk_grants
         (recipe_namespace, recipe_name, capability_family, provider,
          allowed_callers, quota_limits, prompt_targets, default_target_ref,
          policy_revision, policy_state, policy_reviewed_at, policy_reviewed_by)
       VALUES ($1, $2, 'promptBridge', 'openai',
               '["${CALLER_REF}"]'::jsonb, $3::jsonb,
               '[{"targetRef":"primary","provider":"openai","model":"gpt-4o-mini","credentialSlot":"openai-api-key"}]'::jsonb,
               'primary', 1, 'active', now(), 'operator-1')`,
        [NS, recipeName, JSON.stringify(quotaLimits)]
      )
    }

    /**
     * Seed an ACTIVE clientNotifications grant plus the real recipient row the
     * authorizer's `hasUsableClientNotificationRecipients` check resolves
     * against (a `users` row with a non-empty email). Same INSERT mirror as the
     * promptBridge seeding above.
     */
    async function seedClientNotificationsGrant(
      recipeName: string,
      eventType: string,
      quotaLimits: Record<string, number>
    ): Promise<void> {
      const userId = randomUUID()
      await db.pool.query(`INSERT INTO users (id, email) VALUES ($1::uuid, $2)`, [
        userId,
        `sdk-stepless-${recipeName}@e2e.local`,
      ])
      await db.pool.query(
        `INSERT INTO plugin_workload_sdk_grants
         (recipe_namespace, recipe_name, capability_family,
          allowed_event_types, allowed_user_refs, allowed_callers, quota_limits,
          policy_revision, policy_state, policy_reviewed_at, policy_reviewed_by)
       VALUES ($1, $2, 'clientNotifications',
               $3::jsonb, $4::jsonb, '["${CALLER_REF}"]'::jsonb, $5::jsonb,
               1, 'active', now(), 'operator-1')`,
        [
          NS,
          recipeName,
          JSON.stringify([eventType]),
          JSON.stringify([userId]),
          JSON.stringify(quotaLimits),
        ]
      )
    }

    /**
     * Seed invocation-audit rows inside the trailing per-minute window
     * (created_at defaults to now()). contract_version 1 + status 'complete'
     * keeps the rows clear of the v2 lease constraint; the per-row idempotency
     * suffix satisfies the (namespace, name, method, hash) unique index.
     */
    async function seedInvocationWindowRows(
      recipeName: string,
      method: PluginWorkloadSdkFamily,
      detail: string,
      count: number
    ): Promise<void> {
      const result = await db.pool.query(
        `INSERT INTO plugin_workload_sdk_invocations
         (recipe_namespace, recipe_name, caller_ref, method, detail,
          idempotency_key_hash, payload_hash, status, authorization_decision,
          contract_version, attempt_generation)
       SELECT $1, $2, 'stepless-quota-integration', $3, $4,
              $2 || '-' || $3 || '-window-' || g::text, '', 'complete', 'authorized', 1, 1
         FROM generate_series(1, $5::int) AS g`,
        [NS, recipeName, method, detail, count]
      )
      if (result.rowCount !== count) {
        throw new Error(
          `window seed for ${recipeName}/${method} inserted ${result.rowCount} rows, expected ${count}`
        )
      }
    }

    it('never denies a stepless recipe on per-run caps through the AUTHORIZER and books nothing in quota counters (RED vs pre-change code)', async () => {
      const recipeName = `stepless-run-cap-${randomBytes(4).toString('hex')}`
      await assertStepless(recipeName)
      await seedPromptBridgeGrant(recipeName, { maxRequestsPerRun: 3 })
      const claims = claimsFor(recipeName)

      // The deprecated per-run bucket table starts empty for this recipe.
      expect(await quotaCounterRowCount(recipeName), 'counters before any call').toBe(0)

      // FOUR full authorization passes — one PAST the declared per-run cap of 3
      // — through the real enforcement entry point (grant lookup, audit-record
      // reservation, rate check; each with a fresh idempotency key so no replay
      // short-circuit hides the quota path). TARGET behavior: the deprecated
      // per-run cap is inert, so every call is authorized. Pre-change code
      // denied the 4th — the epoch-sentinel bucket (period_start = 1970-01-01)
      // booked every prior call under one immortal per-"run" counter (RED,
      // captured).
      for (let call = 1; call <= 4; call += 1) {
        const result = await authorizer.authorizePromptBridge({
          claims,
          callerRef: CALLER_REF,
          purpose: 'stepless-run-cap-probe',
          idempotencyKey: `stepless-run-cap-${recipeName}-${call}`,
          payload: { call },
        })
        expect(
          result.ok,
          `call ${call}/4: per-run caps are deprecated and the authorizer must not deny (issue #348)${
            result.ok ? '' : ` — got ${result.error}: ${result.message}`
          }`
        ).toBe(true)
      }

      // Proves NO reset is required: even wiping the entire invocation audit
      // trail changes nothing about the decision (pre-change code kept denying
      // because the counter lives in plugin_workload_sdk_quota_counters, which
      // nothing ever resets for a stepless recipe).
      await db.pool.query(
        `DELETE FROM plugin_workload_sdk_invocations
        WHERE recipe_namespace = $1 AND recipe_name = $2`,
        [NS, recipeName]
      )
      const fifth = await authorizer.authorizePromptBridge({
        claims,
        callerRef: CALLER_REF,
        purpose: 'stepless-run-cap-probe',
        idempotencyKey: `stepless-run-cap-${recipeName}-5`,
        payload: { call: 5 },
      })
      expect(fifth.ok, '5th call after deleting every invocation row: still authorized').toBe(true)

      // The epoch-sentinel bucket was never touched: the entire authorization
      // path wrote ZERO rows to plugin_workload_sdk_quota_counters. A future
      // re-wire of the per-run leg inside the authorizer trips this loudly.
      expect(
        await quotaCounterRowCount(recipeName),
        'plugin_workload_sdk_quota_counters must receive no rows from the authorizer path'
      ).toBe(0)
    })

    it('mirrors the per-run-inert contract for clientNotifications through the AUTHORIZER', async () => {
      const recipeName = `stepless-notif-cap-${randomBytes(4).toString('hex')}`
      const eventType = 'lead.followup.due'
      await assertStepless(recipeName)
      await seedClientNotificationsGrant(recipeName, eventType, { maxNotificationsPerRun: 2 })
      const claims = claimsFor(recipeName)

      expect(await quotaCounterRowCount(recipeName), 'counters before any call').toBe(0)

      // THREE notifications — one past the declared per-run cap of 2 — each
      // with a fresh idempotency key. All must be authorized: the per-run cap
      // is deprecated and inert on the real enforcement path.
      for (let call = 1; call <= 3; call += 1) {
        const result = await authorizer.authorizeClientNotification({
          claims,
          callerRef: CALLER_REF,
          eventType,
          idempotencyKey: `stepless-notif-cap-${recipeName}-${call}`,
          payload: { call },
        })
        expect(
          result.ok,
          `notification ${call}/3: per-run caps are deprecated and the authorizer must not deny (issue #348)${
            result.ok ? '' : ` — got ${result.error}: ${result.message}`
          }`
        ).toBe(true)
      }

      expect(
        await quotaCounterRowCount(recipeName),
        'plugin_workload_sdk_quota_counters must receive no rows from the notification authorizer path'
      ).toBe(0)
    })

    it('enforces the per-minute window from the audit trail and frees after 61 seconds (backdated)', async () => {
      // promptBridge at the 600/min platform default: 601 in-window rows deny.
      const bridgeRecipe = `stepless-window-bridge-${randomBytes(4).toString('hex')}`
      await assertStepless(bridgeRecipe)
      await seedPromptBridgeGrant(bridgeRecipe, {})
      const bridgeGrant = await (async () => {
        const sdkDb = await import('../src/services/pluginWorkloadSdkDb.js')
        const grant = await sdkDb.findGrant(NS, bridgeRecipe, 'promptBridge')
        if (!grant) throw new Error(`seeded promptBridge grant not found for ${bridgeRecipe}`)
        return grant
      })()
      await seedInvocationWindowRows(bridgeRecipe, 'promptBridge', 'window-probe', 601)

      await expect(
        quotaTracker.checkRateLimit(NS, bridgeRecipe, 'promptBridge', bridgeGrant)
      ).resolves.toMatchObject({
        ok: false,
        error: 'quota_exceeded',
        retryable: false,
        message: expect.stringContaining('600/minute'),
      })

      // Backdate the whole burst out of the trailing minute (the window clock is
      // Postgres now(), plan D9 R2 — no fake timers, no real waiting) → allowed.
      await db.pool.query(
        `UPDATE plugin_workload_sdk_invocations
          SET created_at = now() - interval '61 seconds'
        WHERE recipe_namespace = $1 AND recipe_name = $2`,
        [NS, bridgeRecipe]
      )
      await expect(
        quotaTracker.checkRateLimit(NS, bridgeRecipe, 'promptBridge', bridgeGrant)
      ).resolves.toEqual({ ok: true })

      // clientNotifications mirror at the 750/min platform default, narrowed to
      // the eventType (detail column).
      const notifRecipe = `stepless-window-notif-${randomBytes(4).toString('hex')}`
      await assertStepless(notifRecipe)
      await seedClientNotificationsGrant(notifRecipe, 'lead.followup.due', {})
      const notifGrant = await (async () => {
        const sdkDb = await import('../src/services/pluginWorkloadSdkDb.js')
        const grant = await sdkDb.findGrant(NS, notifRecipe, 'clientNotifications')
        if (!grant) throw new Error(`seeded clientNotifications grant not found for ${notifRecipe}`)
        return grant
      })()
      await seedInvocationWindowRows(notifRecipe, 'clientNotifications', 'lead.followup.due', 751)

      await expect(
        quotaTracker.checkRateLimit(NS, notifRecipe, 'clientNotifications', notifGrant, {
          eventType: 'lead.followup.due',
        })
      ).resolves.toMatchObject({
        ok: false,
        error: 'quota_exceeded',
        retryable: false,
        message: expect.stringContaining('750/minute'),
      })

      await db.pool.query(
        `UPDATE plugin_workload_sdk_invocations
          SET created_at = now() - interval '61 seconds'
        WHERE recipe_namespace = $1 AND recipe_name = $2`,
        [NS, notifRecipe]
      )
      await expect(
        quotaTracker.checkRateLimit(NS, notifRecipe, 'clientNotifications', notifGrant, {
          eventType: 'lead.followup.due',
        })
      ).resolves.toEqual({ ok: true })
    })

    it('leaves stale epoch-sentinel quota counters inert through the AUTHORIZER — upgraded clusters are freed without migration', async () => {
      const recipeName = `stepless-stale-counter-${randomBytes(4).toString('hex')}`
      await assertStepless(recipeName)
      await seedPromptBridgeGrant(recipeName, { maxRequestsPerRun: 3 })
      const claims = claimsFor(recipeName)

      // A poisoned counter row exactly as a pre-#348 cluster would carry it:
      // booked under the eager sentinel period (epoch) with a count far past
      // any conceivable cap.
      await db.pool.query(
        `INSERT INTO plugin_workload_sdk_quota_counters
         (recipe_namespace, recipe_name, period_start, prompt_bridge_count, notification_count)
       VALUES ($1, $2, to_timestamp(0), 999999, 0)`,
        [NS, recipeName]
      )

      // TARGET behavior: the stale sentinel row is historical data only — it
      // must not deny anything. Pre-change code read it (999999 + 1 > 3) and
      // denied (RED, captured); after Phase 1 the real enforcement entry point
      // never consults plugin_workload_sdk_quota_counters, so upgraded clusters
      // are freed with NO data migration (plan D8: rollout is non-breaking).
      const result = await authorizer.authorizePromptBridge({
        claims,
        callerRef: CALLER_REF,
        purpose: 'stale-sentinel-probe',
        idempotencyKey: `stale-sentinel-${recipeName}-1`,
        payload: { probe: 'stale-sentinel' },
      })
      expect(
        result.ok,
        `stale epoch-sentinel counter must be inert on the authorizer path${
          result.ok ? '' : ` — got ${result.error}: ${result.message}`
        }`
      ).toBe(true)

      // The poisoned row is untouched — historical data only, neither read for
      // enforcement nor written by the authorization pipeline.
      const counter = await db.pool.query<{
        n: number
        prompt_bridge_count: number
        epoch: boolean
      }>(
        `SELECT count(*)::int AS n,
              min(prompt_bridge_count)::int AS prompt_bridge_count,
              bool_and(period_start = to_timestamp(0)) AS epoch
         FROM plugin_workload_sdk_quota_counters
        WHERE recipe_namespace = $1 AND recipe_name = $2`,
        [NS, recipeName]
      )
      expect(counter.rows[0], 'sentinel row must remain exactly as seeded').toEqual({
        n: 1,
        prompt_bridge_count: 999999,
        epoch: true,
      })
    })
  }
)