Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore(admin): remove the autumn-health probe (#5485)
Rakshith Ramprakash8ffe58599538
diff --git a/SELF_HOST.md b/SELF_HOST.mdindex 9fbec7812..8184aa4d6 100644--- a/SELF_HOST.md+++ b/SELF_HOST.md@@ -87,7 +87,7 @@ other variables in the API container's environment.| `ADMIN_INTERNAL_HOST` | `localhost` | Address the internal listener binds to. Bind a wider address only behind a network policy that limits who can reach the port. |Header-authenticated routes: `GET /admin/ops/redis-health`,-`GET /admin/ops/autumn-health`, `POST /admin/ops/acuc-cache-clear`,+`POST /admin/ops/acuc-cache-clear`,`POST /admin/ops/ip-restriction-cache-clear`,`POST /admin/ops/key-restriction-cache-clear` and`POST /admin/ops/concurrency-queue-backfill`.@@ -95,7 +95,7 @@ Header-authenticated routes: `GET /admin/ops/redis-health`,Internal listener routes, none of which need a key: the queue UI at`/admin/queues`; metrics at `GET /admin/metrics`, `GET /admin/nuq-metrics`,`GET /admin/nuq-fdb-metrics` and `GET /admin/index-queue-prometheus`; and-`GET /admin/redis-health`, `GET /admin/autumn-health`, `GET /admin/feng-check`,+`GET /admin/redis-health`, `GET /admin/feng-check`,`GET /admin/precrawl` (queues a precrawl run), `POST /admin/fsearch` (runs asearch) and `POST /admin/crawl-monitor` (still requires an API key and credits).Anyone who can reach the port can use them.diff --git a/apps/api/src/controllers/v0/admin/autumn-health.ts b/apps/api/src/controllers/v0/admin/autumn-health.tsdeleted file mode 100644index 7c074b4ea..000000000--- a/apps/api/src/controllers/v0/admin/autumn-health.ts+++ /dev/null@@ -1,291 +0,0 @@-import { Request, Response } from "express";-import { randomUUID } from "crypto";-import { logger } from "../../../lib/logger";-import { autumnClient } from "../../../services/autumn/client";-import {- CREDITS_FEATURE_ID,- TEAM_FEATURE_ID,-} from "../../../services/autumn/autumn.service";--type StepStatus = "healthy" | "unhealthy" | "skipped";--interface StepResult {- status: StepStatus;- data?: unknown;- error?: string;- durationMs?: number;-}--async function runStep(- name: string,- fn: () => Promise<unknown>,-): Promise<StepResult> {- const start = Date.now();- try {- const data = await fn();- return { status: "healthy", data, durationMs: Date.now() - start };- } catch (error: any) {- logger.error(`Autumn health check step "${name}" failed`, { error });- return {- status: "unhealthy",- error: error.message ?? String(error),- durationMs: Date.now() - start,- };- }-}--export async function autumnHealthController(req: Request, res: Response) {- if (!autumnClient) {- return res.status(503).json({- status: "unhealthy",- message: "Autumn client not configured (AUTUMN_SECRET_KEY missing)",- });- }-- const customerId = `fc-test-${randomUUID()}`;- const entityId = `fc-test-ent-${randomUUID()}`;- const details: Record<string, StepResult> = {};- const totalStart = Date.now();-- try {- // 1. Customer creation (mirrors ensureOrgProvisioned)- details.customerCreate = await runStep("customerCreate", () =>- autumnClient!.customers.getOrCreate({- customerId,- name: "Firecrawl Health Check",- }),- );- if (details.customerCreate.status !== "healthy") {- return respond(res, details, totalStart);- }-- // 2. Entity creation (mirrors ensureTeamProvisioned β uses TEAM feature)- details.entityCreate = await runStep("entityCreate", () =>- autumnClient!.entities.create({- customerId,- entityId,- featureId: TEAM_FEATURE_ID,- }),- );- if (details.entityCreate.status !== "healthy") {- await cleanup(customerId, entityId, details);- return respond(res, details, totalStart);- }-- // 3. Entity read β verify it was created- details.entityGet = await runStep("entityGet", () =>- autumnClient!.entities.get({ customerId, entityId }),- );- if (details.entityGet.status !== "healthy") {- await cleanup(customerId, entityId, details);- return respond(res, details, totalStart);- }-- // 4. Baseline check β snapshot balance before tracking- details.checkBaseline = await runStep("checkBaseline", () =>- autumnClient!.check({- customerId,- entityId,- featureId: CREDITS_FEATURE_ID,- requiredBalance: 0,- }),- );- if (details.checkBaseline.status !== "healthy") {- await cleanup(customerId, entityId, details);- return respond(res, details, totalStart);- }-- // 5. Track +1 credit (mirrors trackCredits)- details.trackCharge = await runStep("trackCharge", () =>- autumnClient!.track({- customerId,- entityId,- featureId: CREDITS_FEATURE_ID,- value: 1,- }),- );- if (details.trackCharge.status !== "healthy") {- await cleanup(customerId, entityId, details);- return respond(res, details, totalStart);- }-- // 6. Check after charge β verify balance moved- details.checkAfterCharge = await runStep("checkAfterCharge", () =>- autumnClient!.check({- customerId,- entityId,- featureId: CREDITS_FEATURE_ID,- requiredBalance: 0,- }),- );-- // 7. Track -1 credit (mirrors refundCredits)- details.trackRefund = await runStep("trackRefund", () =>- autumnClient!.track({- customerId,- entityId,- featureId: CREDITS_FEATURE_ID,- value: -1,- }),- );-- // 8. Check after refund β verify balance restored- details.checkAfterRefund = await runStep("checkAfterRefund", () =>- autumnClient!.check({- customerId,- entityId,- featureId: CREDITS_FEATURE_ID,- requiredBalance: 0,- }),- );-- // 9. Lock credits (mirrors lockCredits β check with lock.enabled)- const lockId = `fc-test-lock-${randomUUID()}`;- details.lockCredits = await runStep("lockCredits", () =>- autumnClient!.check({- customerId,- entityId,- featureId: CREDITS_FEATURE_ID,- requiredBalance: 1,- lock: {- enabled: true,- lockId,- },- }),- );-- // 10. Verify lock was allowed- if (details.lockCredits.status === "healthy") {- const lockData = details.lockCredits.data as any;- if (lockData?.allowed !== true) {- details.lockCredits = {- ...details.lockCredits,- status: "unhealthy",- error: `Lock check returned allowed=${lockData?.allowed}, expected true`,- };- }- }-- // 11. Finalize lock (mirrors finalizeCreditsLock)- if (details.lockCredits.status === "healthy") {- details.finalizeLock = await runStep("finalizeLock", () =>- autumnClient!.balances.finalize({- lockId,- action: "release",- }),- );- }-- // 12. Check after finalize β verify balance unchanged- if (details.finalizeLock?.status === "healthy") {- details.checkAfterFinalize = await runStep("checkAfterFinalize", () =>- autumnClient!.check({- customerId,- entityId,- featureId: CREDITS_FEATURE_ID,- requiredBalance: 0,- }),- );- }-- // 13β14. Cleanup- await cleanup(customerId, entityId, details);-- // Validate all check calls returned allowed- details.checkValidation = validateChecks(details);-- return respond(res, details, totalStart);- } catch (error: any) {- logger.error("Autumn health check unexpected error", { error });- await cleanup(customerId, entityId, details).catch(() => {});- return res.status(500).json({- status: "unhealthy",- message: error.message ?? String(error),- details,- durationMs: Date.now() - totalStart,- });- }-}--function extractRemaining(stepResult: StepResult): number | undefined {- const data = stepResult.data as any;- if (typeof data?.balance?.remaining === "number")- return data.balance.remaining;- return undefined;-}--function validateChecks(details: Record<string, StepResult>): StepResult {- const baseline = extractRemaining(details.checkBaseline);- const afterCharge = extractRemaining(details.checkAfterCharge);- const afterRefund = extractRemaining(details.checkAfterRefund);- const afterFinalize = details.checkAfterFinalize- ? extractRemaining(details.checkAfterFinalize)- : undefined;-- if (- baseline === undefined ||- afterCharge === undefined ||- afterRefund === undefined- ) {- return {- status: "unhealthy",- error: "Could not extract balance.remaining from check responses",- data: { baseline, afterCharge, afterRefund, afterFinalize },- };- }-- const errors: string[] = [];- if (afterCharge !== baseline - 1) {- errors.push(`Charge: expected ${baseline - 1}, got ${afterCharge}`);- }- if (afterRefund !== baseline) {- errors.push(`Refund: expected ${baseline}, got ${afterRefund}`);- }- if (afterFinalize !== undefined && afterFinalize !== baseline) {- errors.push(`Finalize: expected ${baseline}, got ${afterFinalize}`);- }-- return {- status: errors.length === 0 ? "healthy" : "unhealthy",- data: { baseline, afterCharge, afterRefund, afterFinalize },- ...(errors.length > 0 && { error: errors.join("; ") }),- };-}--async function cleanup(- customerId: string,- entityId: string,- details: Record<string, StepResult>,-): Promise<void> {- details.entityDelete = await runStep("entityDelete", () =>- autumnClient!.entities.delete({ customerId, entityId }),- );- details.customerDelete = await runStep("customerDelete", () =>- autumnClient!.customers.delete({ customerId }),- );-}--function respond(- res: Response,- details: Record<string, StepResult>,- totalStart: number,-) {- const allHealthy = Object.values(details).every(- s => s.status === "healthy" || s.status === "skipped",- );-- const status = allHealthy ? "healthy" : "unhealthy";- const httpCode = allHealthy ? 200 : 500;-- if (allHealthy) {- logger.info("Autumn health check passed");- } else {- logger.warn("Autumn health check failed", { details });- }-- return res.status(httpCode).json({- status,- details,- durationMs: Date.now() - totalStart,- });-}diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 5f349ca26..7ce5f99ec 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -2,7 +2,6 @@ import express, { NextFunction, Request, Response } from "express";import { Counter } from "prom-client";import { config } from "../config";import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";-import { autumnHealthController } from "../controllers/v0/admin/autumn-health";import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";import { concurrencyQueueBackfillController } from "../controllers/v0/admin/concurrency-queue-backfill";import { crawlMonitorController } from "../controllers/v0/admin/crawl-monitor";@@ -59,11 +58,6 @@ if (config.BULL_AUTH_KEY && config.BULL_AUTH_KEY_PATH_ROUTES) {redisHealthController,);- adminRouter.get(- `/admin/${config.BULL_AUTH_KEY}/autumn-health`,- autumnHealthController,- );-adminRouter.post(`/admin/${config.BULL_AUTH_KEY}/acuc-cache-clear`,wrap(acucCacheClearController),@@ -133,8 +127,6 @@ if (config.BULL_AUTH_KEY) {opsRouter.get("/redis-health", redisHealthController);- opsRouter.get("/autumn-health", autumnHealthController);-opsRouter.get("/feng-check", wrap(checkFireEngine));opsRouter.post("/acuc-cache-clear", wrap(acucCacheClearController));@@ -212,8 +204,6 @@ export const adminInternalRouter = express.Router();adminInternalRouter.get("/admin/redis-health", redisHealthController);-adminInternalRouter.get("/admin/autumn-health", autumnHealthController);-adminInternalRouter.get("/admin/feng-check", wrap(checkFireEngine));adminInternalRouter.get(diff --git a/apps/api/src/services/autumn/autumn.service.ts b/apps/api/src/services/autumn/autumn.service.tsindex 041787ee8..ca6f64e62 100644--- a/apps/api/src/services/autumn/autumn.service.ts+++ b/apps/api/src/services/autumn/autumn.service.ts@@ -19,7 +19,6 @@ import type {TrackCreditsParams,} from "./types";-export const TEAM_FEATURE_ID = "TEAM";export const CREDITS_FEATURE_ID = "CREDITS";export const SEARCH_CREDITS_FEATURE_ID = "SEARCH_CREDITS";/**