Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add /team/queue-status endpoint (#2063)
Gergő Móricz76cc2decd020
diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.tsindex 2c928b4ac..284dbb6a6 100644--- a/apps/api/src/__tests__/snips/v1/lib.ts+++ b/apps/api/src/__tests__/snips/v1/lib.ts@@ -367,13 +367,16 @@ export async function tokenUsage(identity: Identity): Promise<{ remaining_tokensexport async function concurrencyCheck(identity: Identity): Promise<{ concurrency: number, maxConcurrency: number }> {const x = (await request(TEST_URL)- .get("/v1/concurrency-check")+ .get("/v1/team/queue-status").set("Authorization", `Bearer ${identity.apiKey}`).set("Content-Type", "application/json"));expect(x.statusCode).toBe(200);expect(x.body.success).toBe(true);- return x.body;+ return {+ concurrency: x.body.activeJobsInQueue,+ maxConcurrency: x.body.maxConcurrency,+ };}export async function crawlWithConcurrencyTracking(body: CrawlRequestInput, identity: Identity): Promise<{diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.tsindex ce9e1c1b8..f8a60c772 100644--- a/apps/api/src/__tests__/snips/v2/lib.ts+++ b/apps/api/src/__tests__/snips/v2/lib.ts@@ -311,13 +311,16 @@ export async function creditUsage(identity: Identity): Promise<{ remainingCreditexport async function concurrencyCheck(identity: Identity): Promise<{ concurrency: number, maxConcurrency: number }> {const x = (await request(TEST_URL)- .get("/v2/concurrency-check")+ .get("/v2/team/queue-status").set("Authorization", `Bearer ${identity.apiKey}`).set("Content-Type", "application/json"));expect(x.statusCode).toBe(200);expect(x.body.success).toBe(true);- return x.body;+ return {+ concurrency: x.body.activeJobsInQueue,+ maxConcurrency: x.body.maxConcurrency,+ };}export async function crawlWithConcurrencyTracking(body: CrawlRequestInput, identity: Identity): Promise<{diff --git a/apps/api/src/controllers/v1/queue-status.ts b/apps/api/src/controllers/v1/queue-status.tsnew file mode 100644index 000000000..30f125dd9--- /dev/null+++ b/apps/api/src/controllers/v1/queue-status.ts@@ -0,0 +1,42 @@+import { RateLimiterMode } from "../../types";+import { getACUCTeam } from "../auth";+import { AuthCreditUsageChunkFromTeam, RequestWithAuth } from "./types";+import { Response } from "express";+import { redisEvictConnection } from "../../services/redis";+import { cleanOldConcurrencyLimitedJobs, cleanOldConcurrencyLimitEntries, getConcurrencyLimitActiveJobsCount, getConcurrencyQueueJobsCount } from "../../lib/concurrency-limit";++export type QueueStatusResponse = {+ success: boolean;+ jobsInQueue: number;+ activeJobsInQueue: number;+ waitingJobsInQueue: number;+ maxConcurrency: number;+ mostRecentSuccess: string | null;+}++export async function queueStatusController(req: RequestWithAuth<{}, undefined, QueueStatusResponse>, res: Response<QueueStatusResponse>) {+ let otherACUC: AuthCreditUsageChunkFromTeam | null = null;+ if (!req.acuc?.is_extract) {+ otherACUC = await getACUCTeam(req.auth.team_id, false, true, RateLimiterMode.Extract);+ } else {+ otherACUC = await getACUCTeam(req.auth.team_id, false, true, RateLimiterMode.Crawl);+ }++ await cleanOldConcurrencyLimitEntries(req.auth.team_id);+ const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(req.auth.team_id);+ await cleanOldConcurrencyLimitedJobs(req.auth.team_id);+ const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);++ const mostRecentSuccess = await redisEvictConnection.get("most-recent-success:" + req.auth.team_id);++ return res.status(200).json({+ success: true,++ jobsInQueue: activeJobsOfTeam + queuedJobsOfTeam,+ activeJobsInQueue: activeJobsOfTeam,+ waitingJobsInQueue: queuedJobsOfTeam,+ maxConcurrency: Math.max(req.acuc?.concurrency ?? 1, otherACUC?.concurrency ?? 1),++ mostRecentSuccess: mostRecentSuccess ? new Date(mostRecentSuccess).toISOString() : null,+ });+}\ No newline at end of filediff --git a/apps/api/src/controllers/v2/queue-status.ts b/apps/api/src/controllers/v2/queue-status.tsnew file mode 100644index 000000000..996543b45--- /dev/null+++ b/apps/api/src/controllers/v2/queue-status.ts@@ -0,0 +1,43 @@+import { RateLimiterMode } from "../../types";+import { getACUCTeam } from "../auth";+import { RequestWithAuth } from "./types";+import { AuthCreditUsageChunkFromTeam } from "../v1/types";+import { Response } from "express";+import { redisEvictConnection } from "../../services/redis";+import { cleanOldConcurrencyLimitedJobs, cleanOldConcurrencyLimitEntries, getConcurrencyLimitActiveJobsCount, getConcurrencyQueueJobsCount } from "../../lib/concurrency-limit";++export type QueueStatusResponse = {+ success: boolean;+ jobsInQueue: number;+ activeJobsInQueue: number;+ waitingJobsInQueue: number;+ maxConcurrency: number;+ mostRecentSuccess: string | null;+}++export async function queueStatusController(req: RequestWithAuth<{}, undefined, QueueStatusResponse>, res: Response<QueueStatusResponse>) {+ let otherACUC: AuthCreditUsageChunkFromTeam | null = null;+ if (!req.acuc?.is_extract) {+ otherACUC = await getACUCTeam(req.auth.team_id, false, true, RateLimiterMode.Extract);+ } else {+ otherACUC = await getACUCTeam(req.auth.team_id, false, true, RateLimiterMode.Crawl);+ }++ await cleanOldConcurrencyLimitEntries(req.auth.team_id);+ const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(req.auth.team_id);+ await cleanOldConcurrencyLimitedJobs(req.auth.team_id);+ const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);++ const mostRecentSuccess = await redisEvictConnection.get("most-recent-success:" + req.auth.team_id);++ return res.status(200).json({+ success: true,++ jobsInQueue: activeJobsOfTeam + queuedJobsOfTeam,+ activeJobsInQueue: activeJobsOfTeam,+ waitingJobsInQueue: queuedJobsOfTeam,+ maxConcurrency: Math.max(req.acuc?.concurrency ?? 1, otherACUC?.concurrency ?? 1),++ mostRecentSuccess: mostRecentSuccess ? new Date(mostRecentSuccess).toISOString() : null,+ });+}\ No newline at end of filediff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 14093d2d0..776d7206c 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -20,6 +20,12 @@ export async function cleanOldConcurrencyLimitEntries(await redisEvictConnection.zremrangebyscore(constructKey(team_id), -Infinity, now);}+export async function getConcurrencyLimitActiveJobsCount(+ team_id: string,+): Promise<number> {+ return await redisEvictConnection.zcount(constructKey(team_id), Date.now(), Infinity);+}+export async function getConcurrencyLimitActiveJobs(team_id: string,now: number = Date.now(),@@ -58,10 +64,17 @@ export type ConcurrencyLimitedJob = {priority?: number;};+export async function cleanOldConcurrencyLimitedJobs(+ team_id: string,+ now: number = Date.now(),+) {+ await redisEvictConnection.zremrangebyscore(constructQueueKey(team_id), -Infinity, now);+}+export async function takeConcurrencyLimitedJob(team_id: string,): Promise<ConcurrencyLimitedJob | null> {- await redisEvictConnection.zremrangebyscore(constructQueueKey(team_id), -Infinity, Date.now());+ await cleanOldConcurrencyLimitedJobs(team_id);const res = await redisEvictConnection.zmpop(1, constructQueueKey(team_id), "MIN");if (res === null || res === undefined) {return null;@@ -88,8 +101,7 @@ export async function getConcurrencyLimitedJobs(}export async function getConcurrencyQueueJobsCount(team_id: string): Promise<number> {- const count = await redisEvictConnection.zcard(constructQueueKey(team_id));- return count;+ return await redisEvictConnection.zcount(constructQueueKey(team_id), Date.now(), Infinity);}export async function cleanOldCrawlConcurrencyLimitEntries(diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.tsindex 2e82754e6..16766c3bf 100644--- a/apps/api/src/lib/scrape-billing.ts+++ b/apps/api/src/lib/scrape-billing.ts@@ -1,4 +1,4 @@-import { InternalOptions } from "src/scraper/scrapeURL";+import { InternalOptions } from "../scraper/scrapeURL";import { Document, ScrapeOptions, TeamFlags, shouldParsePDF } from "../controllers/v2/types";import { CostTracking } from "./extract/extraction-service";import { hasFormatOfType } from "./format-utils";diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 018ad304d..ed04065e7 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -33,6 +33,7 @@ import {wrap,} from "./shared";import { paymentMiddleware } from "x402-express";+import { queueStatusController } from "../controllers/v1/queue-status";expressWs(express());@@ -234,9 +235,15 @@ v1Router.get(wrap(tokenUsageController),);+v1Router.get(+ "/team/queue-status",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(queueStatusController),+);+v1Router.post("/x402/search",authMiddleware(RateLimiterMode.Search),countryCheck,wrap(x402SearchController),-);\ No newline at end of file+);diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 0228bed12..896c3781b 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -26,6 +26,7 @@ import {idempotencyMiddleware,wrap,} from "./shared";+import { queueStatusController } from "../controllers/v2/queue-status";expressWs(express());@@ -165,4 +166,10 @@ v2Router.get("/concurrency-check",authMiddleware(RateLimiterMode.CrawlStatus),wrap(concurrencyCheckController),+);++v2Router.get(+ "/team/queue-status",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(queueStatusController),);\ No newline at end of filediff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 668f3a9f6..0e2301582 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -823,6 +823,14 @@ export const processJobInternal = async (job: Job & { id: string }) => {} else {const result = await processJob(job);if (result.success) {+ try {+ if (job.data.team_id) {+ await redisEvictConnection.set("most-recent-success:" + job.data.team_id, new Date().toISOString(), "EX", 60 * 60 * 24);+ }+ } catch (e) {+ logger.warn("Failed to set most recent success", { error: e });+ }+try {if (process.env.USE_DB_AUTHENTICATION === "true" &&diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 313af37ff..9ea0c158f 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.1.0",+ "version": "4.2.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.tsindex 7ea8506f3..e11d7527e 100644--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts@@ -32,5 +32,13 @@ describe("v2.usage e2e", () => {const resp = await client.getTokenUsage();expect(typeof resp.remainingTokens).toBe("number");}, 60_000);++ test("get_queue_status", async () => {+ const resp = await client.getQueueStatus();+ expect(typeof resp.jobsInQueue).toBe("number");+ expect(typeof resp.activeJobsInQueue).toBe("number");+ expect(typeof resp.waitingJobsInQueue).toBe("number");+ expect(typeof resp.maxConcurrency).toBe("number");+ }, 60_000);});diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex eee0397ba..8b5b4b786 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -568,6 +568,22 @@ export interface GenerateLLMsTextStatusResponse {expiresAt: string;}+/**+ * Response interface for queue status operations.+ */+export interface QueueStatusResponse {+ success: boolean;+ jobsInQueue: number;+ activeJobsInQueue: number;+ waitingJobsInQueue: number;+ maxConcurrency: number;++ /**+ * ISO timestamp of the most recent successful scrape in the past 24 hours. Will be null if no successful scrape has occurred in the past 24 hours.+ */+ mostRecentSuccess: string | null;+}+/*** Main class for interacting with the Firecrawl API.* Provides methods for scraping, searching, crawling, and mapping web content.@@ -1586,7 +1602,7 @@ export default class FirecrawlApp {* @param {AxiosResponse} response - The response from the API.* @param {string} action - The action being performed when the error occurred.*/- handleError(response: AxiosResponse, action: string): void {+ handleError(response: AxiosResponse, action: string): never {if (!response) {throw new FirecrawlError(`No response received while trying to ${action}. This may be a network error or the server is unreachable.`,@@ -2027,6 +2043,33 @@ export default class FirecrawlApp {}return { success: false, error: "Internal server error." };}++ /**+ * Gets metrics about the team's scrape queue.+ * @returns The current queue status.+ */+ async getQueueStatus(): Promise<QueueStatusResponse | ErrorResponse> {+ const headers = this.prepareHeaders();+ try {+ const response: AxiosResponse = await this.getRequest(+ `${this.apiUrl}/v1/team/queue-status`,+ headers+ );++ if (response.status === 200) {+ return response.data;+ } else {+ this.handleError(response, "get queue status");+ }+ } catch (error: any) {+ if (error.response?.data?.error) {+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);+ } else {+ throw new FirecrawlError(error.message, 500);+ }+ }+ return { success: false, error: "Internal server error." };+ }}interface CrawlWatcherEvents {diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex 0adce61ba..04f8c2b60 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -19,7 +19,7 @@ import {batchScrape as batchWaiter,} from "./methods/batch";import { startExtract, getExtractStatus, extract as extractWaiter } from "./methods/extract";-import { getConcurrency, getCreditUsage, getTokenUsage } from "./methods/usage";+import { getConcurrency, getCreditUsage, getQueueStatus, getTokenUsage } from "./methods/usage";import type {Document,ScrapeOptions,@@ -269,6 +269,11 @@ export class FirecrawlClient {return getTokenUsage(this.http);}+ /** Metrics about the team's scrape queue. */+ async getQueueStatus() {+ return getQueueStatus(this.http);+ }+// Watcher/*** Create a watcher for a crawl or batch job. Emits: `document`, `snapshot`, `done`, `error`.diff --git a/apps/js-sdk/firecrawl/src/v2/methods/usage.ts b/apps/js-sdk/firecrawl/src/v2/methods/usage.tsindex 37c3f73e0..a3a471df8 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/usage.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/usage.ts@@ -1,4 +1,4 @@-import type { ConcurrencyCheck, CreditUsage, TokenUsage } from "../types";+import type { ConcurrencyCheck, CreditUsage, QueueStatusResponse, TokenUsage } from "../types";import { HttpClient } from "../utils/httpClient";import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";@@ -37,3 +37,13 @@ export async function getTokenUsage(http: HttpClient): Promise<TokenUsage> {}}+export async function getQueueStatus(http: HttpClient): Promise<QueueStatusResponse> {+ try {+ const res = await http.get<QueueStatusResponse>("/v2/team/queue-status");+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get queue status");+ return res.data;+ } catch (err: any) {+ if (err?.isAxiosError) return normalizeAxiosError(err, "get queue status");+ throw err;+ }+}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 1139df6f0..83bddcf4c 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -388,3 +388,11 @@ export class SdkError extends Error {}}+export interface QueueStatusResponse {+ success: boolean;+ jobsInQueue: number;+ activeJobsInQueue: number;+ waitingJobsInQueue: number;+ maxConcurrency: number;+ mostRecentSuccess: string | null;+}diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 35b3983dc..8c47d7995 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -17,7 +17,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.1.1"+__version__ = "4.2.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex a4bc2f60b..c97e5089a 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -727,6 +727,10 @@ class FirecrawlClient:"""Get recent token usage metrics (v2)."""return usage_methods.get_token_usage(self.http_client)+ def get_queue_status(self):+ """Get metrics about the team's scrape queue."""+ return usage_methods.get_queue_status(self.http_client)+def watcher(self,job_id: str,diff --git a/apps/python-sdk/firecrawl/v2/methods/usage.py b/apps/python-sdk/firecrawl/v2/methods/usage.pyindex c9888fe5c..418176780 100644--- a/apps/python-sdk/firecrawl/v2/methods/usage.py+++ b/apps/python-sdk/firecrawl/v2/methods/usage.py@@ -1,5 +1,5 @@from ..utils import HttpClient, handle_response_error-from ..types import ConcurrencyCheck, CreditUsage, TokenUsage+from ..types import ConcurrencyCheck, CreditUsage, QueueStatusResponse, TokenUsagedef get_concurrency(client: HttpClient) -> ConcurrencyCheck:@@ -39,3 +39,18 @@ def get_token_usage(client: HttpClient) -> TokenUsage:remaining_tokens=data.get("remainingTokens", 0))+def get_queue_status(client: HttpClient) -> QueueStatusResponse:+ resp = client.get("/v2/team/queue-status")+ if not resp.ok:+ handle_response_error(resp, "get queue status")+ body = resp.json()+ if not body.get("success"):+ raise Exception(body.get("error", "Unknown error"))+ data = body.get("data", body)+ return QueueStatusResponse(+ jobs_in_queue=data.get("jobsInQueue", 0),+ active_jobs_in_queue=data.get("activeJobsInQueue", 0),+ waiting_jobs_in_queue=data.get("waitingJobsInQueue", 0),+ max_concurrency=data.get("maxConcurrency", 0),+ most_recent_success=data.get("mostRecentSuccess", None),+ )diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 2a56be41a..f3f613b35 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -485,6 +485,14 @@ class TokenUsage(BaseModel):"""Recent token usage metrics (if available)."""remaining_tokens: int+class QueueStatusResponse(BaseModel):+ """Metrics about the team's scrape queue."""+ jobs_in_queue: int+ active_jobs_in_queue: int+ waiting_jobs_in_queue: int+ max_concurrency: int+ most_recent_success: Optional[datetime] = None+# Action typesclass WaitAction(BaseModel):"""Wait action to perform during scraping."""