Skip to content

Evenfire is an intelligence layer platform for running LLM agents that take real actions, on your infrastructure

TypeScript
16.9 KB
Raw Blame
#!/usr/bin/env bash
# Wave-2 Grok subscription T0 aggregator.
#
# Fails when a required suite is missing, executes zero tests, reports
# skipped/todo cases, runs fewer Vitest files than listed, or exits non-zero.
# Exit 0 alone is never enough: the script parses machine-readable counts from
# Vitest or node:test. Every listed suite is guarded by require_file before its
# group runs, because `vitest run <present> <missing>` still exits 0.
#
# Lane separation: Codex-only suites live in test-codex-subscription-t0.sh.
# Real-Postgres suites need CONTROL_API_REAL_PG_ADMIN_URL and run in the CI
# real-PG lane; this script checks they exist and that the lane lists them.
set -u

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
FAIL=0
GROUPS_RUN=0
EXPECTED_GROUPS=11
REGISTERED=()
COUNT_SUMMARY=""

# Read-only inventory mode for hermetic discovery tests and operator inspection.
# It does not enter package, cluster, Electron or browser execution.
if [[ "${1:-}" == "--discovery-only" ]]; then
  if [[ "$#" -ne 2 ]]; then
    echo "usage: $0 --discovery-only <registered-file>" >&2
    exit 2
  fi
  node "${ROOT}/scripts/tests/lib/subscription-t0-discovery.mjs" grok "${ROOT}" "$2"
  exit "$?"
elif [[ "$#" -ne 0 ]]; then
  echo "unsupported T0 argument: $1" >&2
  exit 2
fi

pass() { echo "PASS: $1"; }
fail() { echo "FAIL: $1"; FAIL=1; }

require_file() {
  local rel="$1"
  if [[ ! -f "${ROOT}/${rel}" ]]; then
    fail "missing required suite ${rel}"
    return 1
  fi
  return 0
}

strip_ansi() {
  sed $'s/\033\\[[0-9;]*m//g'
}

# Return 0 when the captured reporter proves at least one passing test, zero
# skipped/todo cases and, for Vitest, exactly the expected number of files.
# Sets COUNT_SUMMARY on success.
assert_executed_counts() {
  local name="$1"
  local log="$2"
  local expected_files="$3"
  local normalized
  normalized="$(mktemp)"
  strip_ansi <"$log" >"$normalized"

  local vitest_passed
  vitest_passed="$(
    grep -Eo 'Tests[[:space:]]+[1-9][0-9]* passed' "$normalized" | head -n 1 | awk '{print $2}' || true
  )"
  local vitest_files
  vitest_files="$(
    grep -Eo 'Test Files[[:space:]]+[0-9]+ passed' "$normalized" | head -n 1 | awk '{print $3}' || true
  )"
  local node_tests
  node_tests="$(
    grep -E '^(#|ℹ) tests[[:space:]]+[0-9]+$' "$normalized" | awk '{print $NF}' | tail -n 1 || true
  )"
  local node_pass
  node_pass="$(
    grep -E '^(#|ℹ) pass[[:space:]]+[1-9][0-9]*$' "$normalized" | awk '{print $NF}' | tail -n 1 || true
  )"

  if [[ -z "${vitest_passed}" && -z "${node_pass}" ]]; then
    fail "${name}: executed zero passing tests"
    echo "----- ${name} reporter -----"
    cat "$normalized"
    rm -f "$normalized"
    return 1
  fi

  if [[ -n "${node_pass}" && "${node_tests}" != "${node_pass}" ]]; then
    fail "${name}: node:test pass count ${node_pass} does not match tests ${node_tests:-missing}"
    rm -f "$normalized"
    return 1
  fi

  if [[ -n "${vitest_passed}" && "${vitest_files}" != "${expected_files}" ]]; then
    fail "${name}: Vitest passed ${vitest_files:-0} test files, expected ${expected_files}"
    grep -E 'Test Files|No test files found' "$normalized" || true
    rm -f "$normalized"
    return 1
  fi

  if grep -Eq '(Test Files|Tests).*[1-9][0-9]* skipped' "$normalized"; then
    fail "${name}: Vitest reported skipped tests"
    rm -f "$normalized"
    return 1
  fi
  if grep -Eq '(Test Files|Tests).*[1-9][0-9]* todo' "$normalized"; then
    fail "${name}: Vitest reported todo tests"
    rm -f "$normalized"
    return 1
  fi
  if grep -Eq '^(#|ℹ) skipped[[:space:]]+[1-9]' "$normalized"; then
    fail "${name}: node:test reported skipped tests"
    rm -f "$normalized"
    return 1
  fi
  if grep -Eq '^(#|ℹ) todo[[:space:]]+[1-9]' "$normalized"; then
    fail "${name}: node:test reported todo tests"
    rm -f "$normalized"
    return 1
  fi
  if grep -Eiq 'no test files|no tests found' "$normalized"; then
    fail "${name}: reporter found no tests"
    rm -f "$normalized"
    return 1
  fi

  if [[ -n "${vitest_passed}" ]]; then
    COUNT_SUMMARY="${vitest_files} files, ${vitest_passed} tests"
  else
    COUNT_SUMMARY="${node_pass}/${node_tests} node:test tests"
  fi
  rm -f "$normalized"
  return 0
}

# run_group <name> <package dir> <suite files relative to the package...>
# Runs the package's own `npm test -- <files>`; the grok-llm-proxy test script
# runs its test typecheck before Vitest.
run_group() {
  local name="$1"
  local prefix="$2"
  shift 2
  if [[ "$#" -eq 0 ]]; then
    fail "${name}: group listed no suite files"
    return 1
  fi
  local files=("$@")
  local rel
  for rel in "${files[@]}"; do
    REGISTERED+=("${prefix}/${rel}")
  done
  for rel in "${files[@]}"; do
    require_file "${prefix}/${rel}" || return 1
  done

  local log
  log="$(mktemp)"
  local npm_args=(--prefix "${ROOT}/${prefix}" test --)
  npm_args+=("${files[@]}")

  echo "── ${name} ──"
  if ! (
    cd "${ROOT}"
    FORCE_COLOR=0 NO_COLOR=1 npm "${npm_args[@]}"
  ) >"$log" 2>&1; then
    fail "${name}: command failed"
    cat "$log"
    rm -f "$log"
    return 1
  fi

  COUNT_SUMMARY=""
  if ! assert_executed_counts "$name" "$log" "${#files[@]}"; then
    rm -f "$log"
    return 1
  fi

  GROUPS_RUN=$((GROUPS_RUN + 1))
  pass "${name} (${COUNT_SUMMARY})"
  rm -f "$log"
}

run_node_group() {
  local name="$1" rel log
  shift
  local files=()
  for rel in "$@"; do
    REGISTERED+=("${rel}")
    require_file "$rel" || return 1
    files+=("${ROOT}/$rel")
  done
  [[ ${#files[@]} -gt 0 ]] || { fail "$name: group listed no suite files"; return 1; }
  log="$(mktemp)"
  echo "── ${name} ──"
  if ! node "$ROOT/scripts/tests/run-node-test-files.mjs" "${files[@]}" >"$log" 2>&1; then
    fail "$name: command failed"
    cat "$log"
    rm -f "$log"
    return 1
  fi
  if ! assert_executed_counts "$name" "$log" "${#files[@]}"; then
    rm -f "$log"
    return 1
  fi
  rm -f "$log"
  GROUPS_RUN=$((GROUPS_RUN + 1))
  pass "$name"
}


require_ci_matrix_entry() {
  local entry="$1"
  if ! grep -Eq "^[[:space:]]+- ${entry}$" "${ROOT}/.github/workflows/ci-public.yml"; then
    fail "ci-public.yml matrix is missing ${entry}"
    return 1
  fi
  pass "ci-public.yml matrix includes ${entry}"
}

# Real-Postgres suites are env-gated (skipped without a real PG), so T0 only
# proves they exist and that the CI real-PG lane asserts each one ran.
require_real_pg_suite() {
  local rel="$1"
  require_file "${rel}" || return 1
  local suite
  # The real-PG lane lists each suite by its file name, `.test.ts` included.
  suite="$(basename "${rel}")"
  if ! sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*\\$//' "${ROOT}/.github/workflows/ci-public.yml" |
    grep -Fxq "${suite}"; then
    fail "ci-public.yml real-PG lane does not list ${suite}"
    return 1
  fi
  pass "real-PG suite present and listed in CI: ${suite}"
}

echo "Grok subscription T0 aggregator"

run_node_group "subscription-discovery-and-offline-image-provider" \
  "scripts/tests/subscription-t0-discovery.test.mjs" \
  "scripts/tests/subscription-image-runner.test.mjs" \
  "scripts/tests/subscription-image-coordinator.test.mjs" \
  "scripts/tests/subscription-image-coordinator-regressions.test.mjs" \
  "scripts/tests/control-api-qa-runtime.test.mjs" \
  "scripts/tests/subscription-private-ui-input.test.mjs" \
  "scripts/tests/subscription-image-input-frames.test.mjs" \
  "scripts/e2e/prepare-subscription-remaining-fixtures.test.mjs" \
  "scripts/e2e/fixtures/subscription-image-provider.test.mjs"

require_ci_matrix_entry "grok-llm-proxy"
require_ci_matrix_entry "packages/grok-provider-attempt-contract"

oauth_source="${ROOT}/control-api/src/services/grokSubscriptionOAuth.ts"
expected_device_url='https://auth.x.ai/oauth2/device/code'
expected_token_url='https://auth.x.ai/oauth2/token'
expected_revoke_url='https://auth.x.ai/oauth2/revoke'
expected_client_id='b1a00492-073a-47ea-816f-4c329264a828'
api_device_url="$(sed -n "s/^export const GROK_OAUTH_DEVICE_URL = '\\(.*\\)'$/\\1/p" "${oauth_source}")"
api_token_url="$(sed -n "s/^export const GROK_OAUTH_TOKEN_URL = '\\(.*\\)'$/\\1/p" "${oauth_source}")"
api_revoke_url="$(sed -n "s/^export const GROK_OAUTH_REVOKE_URL = '\\(.*\\)'$/\\1/p" "${oauth_source}")"
api_client_id="$(
  sed -n "s/.*CONTROL_API_GROK_OAUTH_CLIENT_ID || '\\(.*\\)',$/\\1/p" \
    "${ROOT}/control-api/src/config.ts"
)"
if [[ "${api_device_url}" == "${expected_device_url}" &&
      "${api_token_url}" == "${expected_token_url}" &&
      "${api_revoke_url}" == "${expected_revoke_url}" &&
      "${api_client_id}" == "${expected_client_id}" ]]; then
  pass "Grok OAuth origin/client lock"
else
  fail "Grok OAuth origin/client lock (device=${api_device_url:-missing} token=${api_token_url:-missing} revoke=${api_revoke_url:-missing} client=${api_client_id:-missing})"
fi

run_group "grok-provider-attempt-contract" "packages/grok-provider-attempt-contract" \
  "index.test.cjs"

# Shared projection package; its suite covers the Grok ConfigMap annotations.
run_group "grok-catalog-projection" "packages/codex-catalog-projection" \
  "index.test.cjs"

# Every proxy suite, including the hermetic runtime e2e. The unlisted-suite
# check below fails when a proxy test file is added without being listed here.
run_group "grok-llm-proxy" "grok-llm-proxy" \
  "test/abortWhenClientDisconnects.test.ts" \
  "test/approvedToolsUpstream.test.ts" \
  "test/bindLoopbackSetup.test.ts" \
  "test/bodyAdmission.test.ts" \
  "test/bodyBudget.test.ts" \
  "test/bodyStructure.test.ts" \
  "test/catalogBounds.test.ts" \
  "test/catalogContextWindow.test.ts" \
  "test/contractFreeze.test.ts" \
  "test/controlApiClient.test.ts" \
  "test/deployManifest.test.ts" \
  "test/executionTicketVerifier.test.ts" \
  "test/grokTransport.conformance.test.ts" \
  "test/grokUpstreamHeaders.test.ts" \
  "test/metrics.test.ts" \
  "test/originPolicy.test.ts" \
  "test/redaction.test.ts" \
  "test/releaseAtAcceptance.test.ts" \
  "test/requestLimits.test.ts" \
  "test/runtimePath.hermetic.e2e.test.ts" \
  "test/server.security.test.ts" \
  "test/sseBackpressure.test.ts" \
  "test/sseHeartbeat.test.ts" \
  "test/streamGate.handoff.test.ts" \
  "test/visualBodyReadOwnership.test.ts" \
  "test/toolNameMap.test.ts" \
  "test/upstreamErrorHint.test.ts"

run_group "control-api grok" "control-api" \
  "test/db.grokSubscriptionMigration.test.ts" \
  "test/db.llmProviderAttemptMigration.test.ts" \
  "test/services.grokSubscriptionConnection.test.ts" \
  "test/services.grokSubscriptionOAuth.test.ts" \
  "test/services.grokSubscriptionCatalog.test.ts" \
  "test/services.subscriptionCatalogBounds.test.ts" \
  "test/services.grokProviderAttemptTicket.test.ts" \
  "test/services.grokProviderAttemptRedemption.test.ts" \
  "test/services.llmProviderAttemptAuthorizer.grok.test.ts" \
  "test/services.llmProviderAttemptAuthorizer.depth.test.ts" \
  "test/services.llmProviderAttemptFinalization.test.ts" \
  "test/services.usageEvents.test.ts" \
  "test/services.recipeGrantTransition.test.ts" \
  "test/services.recipeCodexGrantIdentity.test.ts" \
  "test/subscriptionGrantIdentity.test.ts" \
  "test/llmProviders.test.ts" \
  "test/routes.admin.grokSubscription.test.ts" \
  "test/routes.mcp-host.llmProviderAttempts.bodyAdmission.test.ts" \
  "test/llmProviderAttemptBodyAdmission.test.ts" \
  "test/llmProviderAttemptAuthorizer.cancellation.test.ts" \
  "test/helpers.realPostgresCancellation.test.ts" \
  "test/routes.internal.llmProviderAttempts.grok.test.ts" \
  "test/routes.adminPluginWorkloadSdk.test.ts" \
  "test/routes.adminRecipes.test.ts" \
  "test/routes.mcp-host.plugin-workload-sdk.test.ts" \
  "test/hostSpecValidation.grokSubscription.test.ts" \
  "test/hostSpecValidation.limits.test.ts" \
  "test/crd.hostCel.test.ts" \
  "test/crd.llmProviderEnums.test.ts"

require_real_pg_suite "control-api/test/services.grokSubscriptionConnection.realPostgres.integration.test.ts"
require_real_pg_suite "control-api/test/services.grokProviderAttemptRedemption.realPostgres.integration.test.ts"
require_real_pg_suite "control-api/test/services.grokProviderAttemptRedemption.refresh.realPostgres.integration.test.ts"
require_real_pg_suite "control-api/test/pluginWorkloadSdkGrokDualLedger.realPostgres.integration.test.ts"
require_real_pg_suite "control-api/test/db.llmProviderAttemptConnectionIntegrity.realPostgres.integration.test.ts"
# Shared with Codex; it holds the Grok V2 envelope rollback twin (#784).
require_real_pg_suite "control-api/test/services.llmProviderAttemptAuthorization.realPostgres.integration.test.ts"

run_group "mcp-host grok" "mcp-host" \
  "src/llm/__tests__/grokSubscription.test.ts" \
  "src/llm/__tests__/attachmentBudgetRefusal.test.ts" \
  "src/llm/__tests__/imageSource.test.ts" \
  "src/llm/__tests__/grokLlmProxyClient.test.ts" \
  "src/llm/__tests__/imageInput.test.ts" \
  "src/llm/failover/__tests__/subscriptionAdmission.integration.test.ts" \
  "src/llm/__tests__/providerAttemptAuthorizer.test.ts" \
  "src/core/adapters/__tests__/llmImageCompatibility.test.ts" \
  "src/agent/__tests__/taskExecutor.test.ts" \
  "src/llm/__tests__/grokPolicyBinding.test.ts" \
  "src/llm/__tests__/subscriptionRequestHash.test.ts" \
  "src/llm/__tests__/registry.test.ts" \
  "src/llm/__tests__/makeProvider.throws.test.ts" \
  "src/config/configStore.test.ts" \
  "src/core/orchestration/__tests__/toolPresentationPolicy.test.ts" \
  "src/pluginWorkloadSdk/bootstrapIdentity.test.ts" \
  "src/pluginWorkloadSdk/bootstrapServer.test.ts" \
  "src/pluginWorkloadSdk/promptBridge/controlApiClient.test.ts" \
  "src/pluginWorkloadSdk/promptBridge/llmBridge.failover.test.ts"

run_group "workflow-recipes grok" "workflow-recipes" \
  "src/workflow/codexRecipeVerdict.test.ts" \
  "src/workflow/sdkOnlyGrokBinding.test.ts" \
  "src/workflow/pluginWorkloadSdkProvisioner.grokPolicy.test.ts" \
  "src/workflow/workflowReconciler.grokWrcFlag.test.ts" \
  "src/reconciler/workflowRecipeReconciler.grokFlagWiring.test.ts" \
  "src/reconciler/pluginWorkloadSdkValidator.test.ts" \
  "src/mcp/server.pluginSdkBroker.test.ts" \
  "tests/unit/workflow/modelConfigHandler.test.ts" \
  "tests/unit/workflow/modelConfigHandler.pluginSdkBroker.test.ts" \
  "tests/unit/workflow/podFactory.test.ts"

run_group "host-context-controller grok" "host-context-controller" \
  "src/hostReconciler.grokHostRuntime.test.ts" \
  "test/hostReconciler.test.ts"

run_group "control-ui grok" "control-ui" \
  "components/__tests__/CodexSubscriptionHub.test.tsx" \
  "components/__tests__/LlmCatalogFormsGrokGate.test.tsx" \
  "components/__tests__/RecipeEditor.test.tsx" \
  "components/__tests__/HostWizard.test.tsx" \
  "components/__tests__/HostDetailsPage.identity.test.tsx" \
  "components/__tests__/PluginWorkloadSdkPage.test.tsx" \
  "components/__tests__/LlmProviderConfig.test.tsx" \
  "components/__tests__/LlmPolicyEditor.test.tsx" \
  "components/__tests__/LlmModelForm.test.tsx" \
  "lib/__tests__/grokSubscription.sync.test.ts" \
  "lib/__tests__/grokSubscriptionFeature.test.ts" \
  "lib/__tests__/llm.test.ts" \
  "lib/hooks/__tests__/useGrokSubscriptionEnabled.test.tsx"

# The Grok composer budget (16 MiB per image, 16 MiB decoded total, no pixel
# bound) is desktop code. It needs Node 24 and the Electron runtime, exactly as
# the Codex desktop group does; a test result from another runtime is not counted.
node_major=$(node --version | sed -n 's/^v\([0-9][0-9]*\).*/\1/p')
if [[ "${node_major}" != "24" ]]; then
  fail "Desktop T0 requires Node 24.x (got $(node --version))"
else
  pass "Node $(node --version) for Desktop T0"
  if ! (
    cd "${ROOT}/desktop-app"
    npm run verify:electron
  ); then
    fail "desktop-app verify:electron failed"
  else
    pass "desktop-app verify:electron"
    run_group "desktop-app grok" "desktop-app" \
      "test/subscriptionImageCollection.test.ts" \
      "test/subscriptionImageChallenge.test.ts" \
      "test/subscriptionImageRunContract.test.ts" \
      "test/subscriptionAdmissionGuard.test.ts" \
      "test/subscriptionRemainingJourneyData.test.ts" \
      "test/subscriptionRemainingJourneysContract.test.ts" \
      "ui/src/constants/__tests__/attachments.test.ts" \
      "ui/src/components/agents/__tests__/ComposerPanel.test.tsx"
  fi
fi

echo "── grok-llm-proxy deploy contract ──"
if require_file "scripts/tests/test-grok-llm-proxy-deploy-contract.sh" &&
   require_file "tests/e2e/fixtures/grok-subscription/sanitized-upstream-contract.json"; then
  if bash "${ROOT}/scripts/tests/test-grok-llm-proxy-deploy-contract.sh"; then
    GROUPS_RUN=$((GROUPS_RUN + 1))
    pass "grok-llm-proxy deploy contract"
  else
    fail "grok-llm-proxy deploy contract"
  fi
fi

# Every candidate is registered in T0 or has an explicit CI/PG/runtime lane.
# Discovery never certifies those other lanes; each requires its own physical receipt.
discovery_registry="$(mktemp)"
printf '%s\n' "${REGISTERED[@]}" >"${discovery_registry}"
if ! node "${ROOT}/scripts/tests/lib/subscription-t0-discovery.mjs" grok "${ROOT}" "${discovery_registry}"; then
  FAIL=1
fi
rm -f "${discovery_registry}"

if [[ "${GROUPS_RUN}" -ne "${EXPECTED_GROUPS}" ]]; then
  fail "expected all ${EXPECTED_GROUPS} T0 groups, ran ${GROUPS_RUN}"
fi

if [[ "${FAIL}" -ne 0 ]]; then
  echo "FAIL: grok subscription T0"
  exit 1
fi
echo "PASS: grok subscription T0 (${GROUPS_RUN} groups)"
exit 0