Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
15.3 KB
Raw Blame
// Deterministic E2E plugin workload for the Plugin Workload SDK.
//
// Reads the recipe-injected env (PLUGIN_WORKLOAD_SDK_ENDPOINT +
// PLUGIN_WORKLOAD_SDK_TOKEN), exercises promptBridge and clientNotifications,
// then validates idempotency replay and quota enforcement.
// Prints structured RESULT lines the E2E gate greps for, then stays alive
// so the gate can inspect the pod. Node 24 provides a global fetch — this
// image has zero npm dependencies on purpose (reproducible).
//
// Markers the gate asserts on:
//   E2E_SDK_PROMPT_BRIDGE_OK=<invocationId>
//   E2E_SDK_PROMPT_BRIDGE_FAIL=<code>
//   E2E_SDK_CLIENT_NOTIFICATION_OK=<notificationId>
//   E2E_SDK_CLIENT_NOTIFICATION_FAIL=<code>
//   E2E_SDK_IDEMPOTENCY_REPLAY_GUARDED          (replay is rejected without a second provider call)
//   E2E_SDK_EXPLICIT_TARGET_OK=<invocationId>   (approved targetRef was served)
//   E2E_SDK_IDEMPOTENCY_FAIL=<reason>
//   E2E_SDK_QUOTA_EXCEEDED_OK                    (N+1 call correctly rejected)
//   E2E_SDK_QUOTA_EXCEEDED_FAIL=<reason>
//   E2E_SDK_DONE
import http from 'node:http'

const ENDPOINT = process.env.PLUGIN_WORKLOAD_SDK_ENDPOINT || ''
const TOKEN = process.env.PLUGIN_WORKLOAD_SDK_TOKEN || ''
const CALLER_REF = process.env.E2E_SDK_CALLER_REF || 'sdk-caller'
const EVENT_TYPE = process.env.E2E_SDK_EVENT_TYPE || 'e2e.test.notification'
const USER_REF = process.env.E2E_SDK_USER_REF || 'e2e-test-user'
const RUN_ID = process.env.E2E_SDK_RUN_ID || String(Date.now())
const QUOTA_LIMIT = parseInt(process.env.E2E_SDK_QUOTA_LIMIT || '4', 10)
const EXPLICIT_TARGET_REF = process.env.E2E_SDK_EXPLICIT_TARGET_REF || ''
const SANDBOX_UI_MAX_ATTEMPTS = parseInt(process.env.E2E_SDK_SANDBOX_UI_MAX_ATTEMPTS || '10', 10)
// Every SDK request must fail closed if the server stops responding.  The
// outer shell gate also has bounded polling, but fetch itself otherwise has no
// implicit deadline and could keep a fixture alive indefinitely.
const SDK_REQUEST_TIMEOUT_MS = 15_000

/** Tracks the invocationId from the first promptBridge call for idempotency check. */
let firstPromptBridgeId = null

function log(line) {
  process.stdout.write(`${line}\n`)
}

const sleep = ms => new Promise(resolve => setTimeout(resolve, ms))

async function callSdk(path, body) {
  const res = await fetch(`${ENDPOINT}${path}`, {
    method: 'POST',
    headers: {
      'Content-Type': 'application/json',
      Authorization: `Bearer ${TOKEN}`,
      'x-clerum-caller-ref': CALLER_REF,
    },
    body: JSON.stringify(body),
    signal: AbortSignal.timeout(SDK_REQUEST_TIMEOUT_MS),
  })
  let parsed = null
  try {
    parsed = await res.json()
  } catch {
    parsed = null
  }
  return { status: res.status, body: parsed }
}

async function exercisePromptBridge() {
  // Two distinct cold-start races are retryable here, and a real plugin workload
  // would survive both rather than fail on a cold start:
  //   1. provider_unavailable — the SDK server (:8099) answers, but the WRC has
  //      not yet brokered the provider into the eager mcp-host on its
  //      (level-triggered) reconcile loop, which may land a few seconds later.
  //   2. transport exception (e.g. "fetch failed") — the SDK listener itself, or
  //      its Service endpoints right after the caller pod is recreated, is not
  //      yet accepting connections. fetch() throws before any HTTP status.
  // The eager mcp-host is brokered the provider only after it boots AND becomes
  // Ready AND the WRC's level-triggered reconcile fires the eager /configure —
  // ~60-90s from recipe apply on a cold cluster, not "a few seconds". Retry wide
  // enough to outlast that window; a real plugin workload would do the same.
  const MAX_ATTEMPTS = 30
  const RETRY_DELAY_MS = 3000
  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
    try {
      const { status, body } = await callSdk('/v1/prompt-bridge', {
        purpose: 'summarization',
        idempotencyKey: `e2e-prompt-${RUN_ID}`,
        messages: [{ role: 'user', content: 'Summarize: the quick brown fox.' }],
      })
      if (status === 200 && body && body.invocationId) {
        firstPromptBridgeId = body.invocationId
        log(`E2E_SDK_PROMPT_BRIDGE_OK=${body.invocationId}`)
        return
      }
      const code = (body && (body.error || body.code)) || `http_${status}`
      if (code === 'provider_unavailable' && attempt < MAX_ATTEMPTS) {
        await sleep(RETRY_DELAY_MS)
        continue
      }
      log(`E2E_SDK_PROMPT_BRIDGE_FAIL=${code}`)
      return
    } catch (err) {
      // Transport-level cold start: retry through the same window as case (1).
      if (attempt < MAX_ATTEMPTS) {
        await sleep(RETRY_DELAY_MS)
        continue
      }
      log(`E2E_SDK_PROMPT_BRIDGE_FAIL=exception:${err && err.message ? err.message : err}`)
      return
    }
  }
}

async function exerciseClientNotification() {
  // Same transport cold-start tolerance as promptBridge: a recreated caller pod
  // can race the SDK listener / Service endpoints. Retry the connection-level
  // failure rather than declaring a hard FAIL on the first refused connection.
  const MAX_ATTEMPTS = 10
  const RETRY_DELAY_MS = 3000
  for (let attempt = 1; attempt <= MAX_ATTEMPTS; attempt++) {
    try {
      const { status, body } = await callSdk('/v1/client-notifications', {
        eventType: EVENT_TYPE,
        userRef: USER_REF,
        idempotencyKey: `e2e-notify-${RUN_ID}`,
        notification: {
          title: 'E2E notification',
          body: 'Sent by the plugin workload SDK E2E fixture.',
        },
      })
      if (status === 200 && body && body.notificationId) {
        log(`E2E_SDK_CLIENT_NOTIFICATION_OK=${body.notificationId}`)
      } else {
        const code = (body && (body.error || body.code)) || `http_${status}`
        log(`E2E_SDK_CLIENT_NOTIFICATION_FAIL=${code}`)
      }
      return
    } catch (err) {
      if (attempt < MAX_ATTEMPTS) {
        await sleep(RETRY_DELAY_MS)
        continue
      }
      log(`E2E_SDK_CLIENT_NOTIFICATION_FAIL=exception:${err && err.message ? err.message : err}`)
      return
    }
  }
}

/**
 * The mounted Sandbox UI fixture uses the same production SDK client-notification
 * route as the background workload. Keeping this in the fixture image lets the
 * Desktop E2E establish its notification from the app it actually mounts.
 */
async function emitSandboxUiNotification() {
  for (let attempt = 1; attempt <= SANDBOX_UI_MAX_ATTEMPTS; attempt++) {
    try {
      const { status, body } = await callSdk('/v1/client-notifications', {
        eventType: EVENT_TYPE,
        userRef: USER_REF,
        idempotencyKey: `e2e-sandbox-ui-notify-${RUN_ID}`,
        notification: {
          title: 'E2E Sandbox UI notification',
          body: 'Sent by the mounted Plugin Workload SDK Sandbox UI fixture.',
        },
      })
      if (status === 200 && body && body.notificationId) {
        log(`E2E_SDK_SANDBOX_UI_NOTIFICATION_OK=${body.notificationId}`)
        return body.notificationId
      }
      const code = (body && (body.error || body.code)) || `http_${status}`
      if (attempt === SANDBOX_UI_MAX_ATTEMPTS) {
        throw new Error(`client notification failed: ${code}`)
      }
    } catch (err) {
      if (attempt === SANDBOX_UI_MAX_ATTEMPTS) throw err
    }
    await sleep(3000)
  }
  throw new Error('sandbox-ui notification retry loop exhausted')
}

function startSandboxUiFixture() {
  let notification = null
  const server = http.createServer(async (req, res) => {
    if (req.url === '/healthz') {
      res.writeHead(200, { 'content-type': 'text/plain' })
      res.end('ok')
      return
    }
    notification ??= emitSandboxUiNotification()
    try {
      const notificationId = await notification
      res.writeHead(200, { 'content-type': 'text/html; charset=utf-8' })
      res.end(
        `<!doctype html><title>E2E Layout Notification App</title><main><h1>E2E Layout Notification App</h1><p data-notification-id="${notificationId}">Notification sent.</p></main>`
      )
    } catch (error) {
      const reason =
        error instanceof Error &&
        /^client notification failed: (?:[a-z0-9_-]+|http_\d+)$/i.test(error.message)
          ? error.message.replace('client notification failed: ', '')
          : 'notification_emit_failed'
      log(`E2E_SDK_SANDBOX_UI_FAIL=${reason}`)
      res.writeHead(500, { 'content-type': 'text/plain; charset=utf-8' })
      res.end('Could not emit Sandbox UI notification.')
    }
  })
  server.listen(Number(process.env.PORT || '8080'), '0.0.0.0', () => {
    log('E2E_SDK_SANDBOX_UI_READY')
  })
}

// Explicit selector: the backend can request one target from the operator's
// ordered policy. The targetRef is injected as non-secret recipe configuration;
// the workload never receives a provider key or credential slot value.
async function exerciseExplicitTarget() {
  if (!EXPLICIT_TARGET_REF) {
    log('E2E_SDK_EXPLICIT_TARGET_FAIL=missing_target_ref')
    return
  }
  try {
    const { status, body } = await callSdk('/v1/prompt-bridge', {
      purpose: 'generation',
      targetRef: EXPLICIT_TARGET_REF,
      idempotencyKey: `e2e-explicit-${RUN_ID}`,
      messages: [{ role: 'user', content: 'Reply with the word approved.' }],
    })
    if (
      status === 200 &&
      body &&
      body.invocationId &&
      typeof body.content === 'string' &&
      body.content.trim() &&
      body.servedTarget &&
      body.servedTarget.targetRef === EXPLICIT_TARGET_REF
    ) {
      log(`E2E_SDK_EXPLICIT_TARGET_OK=${body.invocationId}`)
    } else {
      const code = (body && (body.error || body.code)) || `http_${status}`
      log(`E2E_SDK_EXPLICIT_TARGET_FAIL=${code}`)
    }
  } catch (err) {
    log(`E2E_SDK_EXPLICIT_TARGET_FAIL=exception:${err && err.message ? err.message : err}`)
  }
}

// Idempotency: replay the same idempotencyKey must never invoke the provider or
// consume another quota slot. The public mcp-host route deliberately returns
// idempotency_conflict for a completed prompt because the audit record does not
// persist completion content to replay. The authorization record still keeps
// the original invocation identity; callers must use a fresh key when they
// need a new completion.
async function exerciseIdempotency() {
  try {
    const { status, body } = await callSdk('/v1/prompt-bridge', {
      purpose: 'summarization',
      idempotencyKey: `e2e-prompt-${RUN_ID}`, // SAME key as first call
      messages: [{ role: 'user', content: 'Summarize: the quick brown fox.' }],
    })
    const code = (body && (body.error || body.code)) || `http_${status}`
    if (status === 422 && code === 'idempotency_conflict') {
      log('E2E_SDK_IDEMPOTENCY_REPLAY_GUARDED')
    } else if (status === 200 && body && body.invocationId === firstPromptBridgeId) {
      // Keep accepting the richer replay response if the SDK later adds
      // durable completion replay without changing the no-double-charge
      // invariant.
      log('E2E_SDK_IDEMPOTENCY_REPLAY_GUARDED')
    } else if (status === 200 && body && body.invocationId !== firstPromptBridgeId) {
      log(`E2E_SDK_IDEMPOTENCY_FAIL=different_invocation_id:${body.invocationId}`)
    } else {
      log(`E2E_SDK_IDEMPOTENCY_FAIL=${code}`)
    }
  } catch (err) {
    log(`E2E_SDK_IDEMPOTENCY_FAIL=exception:${err && err.message ? err.message : err}`)
  }
}

// Quota enforcement: fill remaining quota slots then verify N+1 is rejected.
// The default and explicit-target prompt calls consumed 2 slots; the
// idempotency replay does NOT consume a slot. Fill the remaining slots, then
// attempt one beyond.
//
// PER-MINUTE WINDOW CAVEAT (issue #348): enforcement moved from a per-run
// counter (no clock) to a trailing 60s window. Reusing the Phase-1 happy call
// and the Phase-2 explicit call as 2 of the QUOTA_LIMIT slots means those two
// rows must still be inside the 60s window when the exceed call fires. Rows are
// timestamped record-first (call start), but the SPACING between them absorbs
// real provider latency (up to ~15s/promptBridge fetch + notification
// transport). If the happy row ages past 60s before the exceed call, the window
// count drops below the limit and the exceed call is wrongly ACCEPTED — a
// spurious RED (never a false pass). A deterministic fix requires the gate to
// reset invocation rows immediately before a tight, self-contained quota burst
// so earlier phases can't age out of the window; that coordinated gate+fixture
// change must be validated on a live minikube cert run (deferred). Until then
// the FAIL reasons below name the window hypothesis so a spurious RED is
// diagnosable at a glance.
async function exerciseQuotaEnforcement() {
  try {
    const remaining = QUOTA_LIMIT - 2
    for (let i = 0; i < remaining; i++) {
      const { status, body } = await callSdk('/v1/prompt-bridge', {
        purpose: 'classification',
        idempotencyKey: `e2e-quota-fill-${i}-${RUN_ID}`,
        messages: [{ role: 'user', content: `Classify: test item ${i}.` }],
      })
      if (status !== 200) {
        const code = (body && (body.error || body.code)) || `http_${status}`
        // A quota/429 here means an EARLIER phase's rows plus these fills already
        // crossed the per-minute limit within the 60s window (window over-count).
        const hint = code.includes('quota') || status === 429 ? ':window_overcount' : ''
        log(`E2E_SDK_QUOTA_EXCEEDED_FAIL=fill_call_failed:${code}${hint}`)
        return
      }
    }

    // Quota should now be exhausted. The next call must be rejected.
    const { status, body } = await callSdk('/v1/prompt-bridge', {
      purpose: 'classification',
      idempotencyKey: `e2e-quota-exceed-${RUN_ID}`,
      messages: [{ role: 'user', content: 'This should be rejected by quota.' }],
    })
    if (status !== 200) {
      const code = (body && (body.error || body.code)) || `http_${status}`
      if (code.includes('quota') || status === 429) {
        log('E2E_SDK_QUOTA_EXCEEDED_OK')
      } else {
        log(`E2E_SDK_QUOTA_EXCEEDED_FAIL=wrong_error:${code}`)
      }
    } else {
      // Accepted when it should have been denied. Under per-minute semantics the
      // likeliest cause is an earlier filler row aging past the 60s window
      // before this call, so the in-window count fell below the limit.
      log('E2E_SDK_QUOTA_EXCEEDED_FAIL=call_accepted_after_quota_exhausted:possible_window_ageout')
    }
  } catch (err) {
    log(`E2E_SDK_QUOTA_EXCEEDED_FAIL=exception:${err && err.message ? err.message : err}`)
  }
}

async function main() {
  if (process.env.E2E_SDK_MODE === 'sandbox-ui') {
    if (!ENDPOINT || !TOKEN) {
      log('E2E_SDK_SANDBOX_UI_FAIL=missing_endpoint_or_token')
      return
    }
    startSandboxUiFixture()
    return
  }
  if (!ENDPOINT || !TOKEN) {
    log('E2E_SDK_PROMPT_BRIDGE_FAIL=missing_endpoint_or_token')
    log('E2E_SDK_CLIENT_NOTIFICATION_FAIL=missing_endpoint_or_token')
  } else {
    // Phase 1: Happy path — one promptBridge + one clientNotification.
    await exercisePromptBridge()
    await exerciseClientNotification()

    // Phase 1b: explicit approved target selection.
    if (firstPromptBridgeId) {
      await exerciseExplicitTarget()
    }

    // Phase 2: Idempotency — same key returns same invocationId.
    if (firstPromptBridgeId) {
      await exerciseIdempotency()
    }

    // Phase 3: Quota enforcement — fill quota then verify rejection.
    if (firstPromptBridgeId) {
      await exerciseQuotaEnforcement()
    }
  }
  log('E2E_SDK_DONE')
  // Stay alive so the E2E gate can read logs and inspect the pod env.
  setInterval(() => {}, 1 << 30)
}

void main()