Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add /team/queue-status endpoint (#2063)

Gergő Móricz76cc2decd020
diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.ts
index 2c928b4ac..284dbb6a6 100644
--- a/apps/api/src/__tests__/snips/v1/lib.ts
+++ b/apps/api/src/__tests__/snips/v1/lib.ts
@@ -367,13 +367,16 @@ export async function tokenUsage(identity: Identity): Promise<{ remaining_tokens
export async function concurrencyCheck(identity: Identity): Promise<{ concurrency: number, maxConcurrency: number }> {
const x = (await request(TEST_URL)
- .get("/v1/concurrency-check")
+ .get("/v1/team/queue-status")
.set("Authorization", `Bearer ${identity.apiKey}`)
.set("Content-Type", "application/json"));
expect(x.statusCode).toBe(200);
expect(x.body.success).toBe(true);
- return x.body;
+ return {
+ concurrency: x.body.activeJobsInQueue,
+ maxConcurrency: x.body.maxConcurrency,
+ };
}
export async function crawlWithConcurrencyTracking(body: CrawlRequestInput, identity: Identity): Promise<{
diff --git a/apps/api/src/__tests__/snips/v2/lib.ts b/apps/api/src/__tests__/snips/v2/lib.ts
index ce9e1c1b8..f8a60c772 100644
--- a/apps/api/src/__tests__/snips/v2/lib.ts
+++ b/apps/api/src/__tests__/snips/v2/lib.ts
@@ -311,13 +311,16 @@ export async function creditUsage(identity: Identity): Promise<{ remainingCredit
export async function concurrencyCheck(identity: Identity): Promise<{ concurrency: number, maxConcurrency: number }> {
const x = (await request(TEST_URL)
- .get("/v2/concurrency-check")
+ .get("/v2/team/queue-status")
.set("Authorization", `Bearer ${identity.apiKey}`)
.set("Content-Type", "application/json"));
expect(x.statusCode).toBe(200);
expect(x.body.success).toBe(true);
- return x.body;
+ return {
+ concurrency: x.body.activeJobsInQueue,
+ maxConcurrency: x.body.maxConcurrency,
+ };
}
export async function crawlWithConcurrencyTracking(body: CrawlRequestInput, identity: Identity): Promise<{
diff --git a/apps/api/src/controllers/v1/queue-status.ts b/apps/api/src/controllers/v1/queue-status.ts
new file mode 100644
index 000000000..30f125dd9
--- /dev/null
+++ b/apps/api/src/controllers/v1/queue-status.ts
@@ -0,0 +1,42 @@
+import { RateLimiterMode } from "../../types";
+import { getACUCTeam } from "../auth";
+import { AuthCreditUsageChunkFromTeam, RequestWithAuth } from "./types";
+import { Response } from "express";
+import { redisEvictConnection } from "../../services/redis";
+import { cleanOldConcurrencyLimitedJobs, cleanOldConcurrencyLimitEntries, getConcurrencyLimitActiveJobsCount, getConcurrencyQueueJobsCount } from "../../lib/concurrency-limit";
+
+export type QueueStatusResponse = {
+ success: boolean;
+ jobsInQueue: number;
+ activeJobsInQueue: number;
+ waitingJobsInQueue: number;
+ maxConcurrency: number;
+ mostRecentSuccess: string | null;
+}
+
+export async function queueStatusController(req: RequestWithAuth<{}, undefined, QueueStatusResponse>, res: Response<QueueStatusResponse>) {
+ let otherACUC: AuthCreditUsageChunkFromTeam | null = null;
+ if (!req.acuc?.is_extract) {
+ otherACUC = await getACUCTeam(req.auth.team_id, false, true, RateLimiterMode.Extract);
+ } else {
+ otherACUC = await getACUCTeam(req.auth.team_id, false, true, RateLimiterMode.Crawl);
+ }
+
+ await cleanOldConcurrencyLimitEntries(req.auth.team_id);
+ const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(req.auth.team_id);
+ await cleanOldConcurrencyLimitedJobs(req.auth.team_id);
+ const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
+
+ const mostRecentSuccess = await redisEvictConnection.get("most-recent-success:" + req.auth.team_id);
+
+ return res.status(200).json({
+ success: true,
+
+ jobsInQueue: activeJobsOfTeam + queuedJobsOfTeam,
+ activeJobsInQueue: activeJobsOfTeam,
+ waitingJobsInQueue: queuedJobsOfTeam,
+ maxConcurrency: Math.max(req.acuc?.concurrency ?? 1, otherACUC?.concurrency ?? 1),
+
+ mostRecentSuccess: mostRecentSuccess ? new Date(mostRecentSuccess).toISOString() : null,
+ });
+}
\ No newline at end of file
diff --git a/apps/api/src/controllers/v2/queue-status.ts b/apps/api/src/controllers/v2/queue-status.ts
new file mode 100644
index 000000000..996543b45
--- /dev/null
+++ b/apps/api/src/controllers/v2/queue-status.ts
@@ -0,0 +1,43 @@
+import { RateLimiterMode } from "../../types";
+import { getACUCTeam } from "../auth";
+import { RequestWithAuth } from "./types";
+import { AuthCreditUsageChunkFromTeam } from "../v1/types";
+import { Response } from "express";
+import { redisEvictConnection } from "../../services/redis";
+import { cleanOldConcurrencyLimitedJobs, cleanOldConcurrencyLimitEntries, getConcurrencyLimitActiveJobsCount, getConcurrencyQueueJobsCount } from "../../lib/concurrency-limit";
+
+export type QueueStatusResponse = {
+ success: boolean;
+ jobsInQueue: number;
+ activeJobsInQueue: number;
+ waitingJobsInQueue: number;
+ maxConcurrency: number;
+ mostRecentSuccess: string | null;
+}
+
+export async function queueStatusController(req: RequestWithAuth<{}, undefined, QueueStatusResponse>, res: Response<QueueStatusResponse>) {
+ let otherACUC: AuthCreditUsageChunkFromTeam | null = null;
+ if (!req.acuc?.is_extract) {
+ otherACUC = await getACUCTeam(req.auth.team_id, false, true, RateLimiterMode.Extract);
+ } else {
+ otherACUC = await getACUCTeam(req.auth.team_id, false, true, RateLimiterMode.Crawl);
+ }
+
+ await cleanOldConcurrencyLimitEntries(req.auth.team_id);
+ const activeJobsOfTeam = await getConcurrencyLimitActiveJobsCount(req.auth.team_id);
+ await cleanOldConcurrencyLimitedJobs(req.auth.team_id);
+ const queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
+
+ const mostRecentSuccess = await redisEvictConnection.get("most-recent-success:" + req.auth.team_id);
+
+ return res.status(200).json({
+ success: true,
+
+ jobsInQueue: activeJobsOfTeam + queuedJobsOfTeam,
+ activeJobsInQueue: activeJobsOfTeam,
+ waitingJobsInQueue: queuedJobsOfTeam,
+ maxConcurrency: Math.max(req.acuc?.concurrency ?? 1, otherACUC?.concurrency ?? 1),
+
+ mostRecentSuccess: mostRecentSuccess ? new Date(mostRecentSuccess).toISOString() : null,
+ });
+}
\ No newline at end of file
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 14093d2d0..776d7206c 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -20,6 +20,12 @@ export async function cleanOldConcurrencyLimitEntries(
await redisEvictConnection.zremrangebyscore(constructKey(team_id), -Infinity, now);
}
+export async function getConcurrencyLimitActiveJobsCount(
+ team_id: string,
+): Promise<number> {
+ return await redisEvictConnection.zcount(constructKey(team_id), Date.now(), Infinity);
+}
+
export async function getConcurrencyLimitActiveJobs(
team_id: string,
now: number = Date.now(),
@@ -58,10 +64,17 @@ export type ConcurrencyLimitedJob = {
priority?: number;
};
+export async function cleanOldConcurrencyLimitedJobs(
+ team_id: string,
+ now: number = Date.now(),
+) {
+ await redisEvictConnection.zremrangebyscore(constructQueueKey(team_id), -Infinity, now);
+}
+
export async function takeConcurrencyLimitedJob(
team_id: string,
): Promise<ConcurrencyLimitedJob | null> {
- await redisEvictConnection.zremrangebyscore(constructQueueKey(team_id), -Infinity, Date.now());
+ await cleanOldConcurrencyLimitedJobs(team_id);
const res = await redisEvictConnection.zmpop(1, constructQueueKey(team_id), "MIN");
if (res === null || res === undefined) {
return null;
@@ -88,8 +101,7 @@ export async function getConcurrencyLimitedJobs(
}
export async function getConcurrencyQueueJobsCount(team_id: string): Promise<number> {
- const count = await redisEvictConnection.zcard(constructQueueKey(team_id));
- return count;
+ return await redisEvictConnection.zcount(constructQueueKey(team_id), Date.now(), Infinity);
}
export async function cleanOldCrawlConcurrencyLimitEntries(
diff --git a/apps/api/src/lib/scrape-billing.ts b/apps/api/src/lib/scrape-billing.ts
index 2e82754e6..16766c3bf 100644
--- a/apps/api/src/lib/scrape-billing.ts
+++ b/apps/api/src/lib/scrape-billing.ts
@@ -1,4 +1,4 @@
-import { InternalOptions } from "src/scraper/scrapeURL";
+import { InternalOptions } from "../scraper/scrapeURL";
import { Document, ScrapeOptions, TeamFlags, shouldParsePDF } from "../controllers/v2/types";
import { CostTracking } from "./extract/extraction-service";
import { hasFormatOfType } from "./format-utils";
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index 018ad304d..ed04065e7 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -33,6 +33,7 @@ import {
wrap,
} from "./shared";
import { paymentMiddleware } from "x402-express";
+import { queueStatusController } from "../controllers/v1/queue-status";
expressWs(express());
@@ -234,9 +235,15 @@ v1Router.get(
wrap(tokenUsageController),
);
+v1Router.get(
+ "/team/queue-status",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(queueStatusController),
+);
+
v1Router.post(
"/x402/search",
authMiddleware(RateLimiterMode.Search),
countryCheck,
wrap(x402SearchController),
-);
\ No newline at end of file
+);
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 0228bed12..896c3781b 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -26,6 +26,7 @@ import {
idempotencyMiddleware,
wrap,
} from "./shared";
+import { queueStatusController } from "../controllers/v2/queue-status";
expressWs(express());
@@ -165,4 +166,10 @@ v2Router.get(
"/concurrency-check",
authMiddleware(RateLimiterMode.CrawlStatus),
wrap(concurrencyCheckController),
+);
+
+v2Router.get(
+ "/team/queue-status",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(queueStatusController),
);
\ No newline at end of file
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 668f3a9f6..0e2301582 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -823,6 +823,14 @@ export const processJobInternal = async (job: Job & { id: string }) => {
} else {
const result = await processJob(job);
if (result.success) {
+ try {
+ if (job.data.team_id) {
+ await redisEvictConnection.set("most-recent-success:" + job.data.team_id, new Date().toISOString(), "EX", 60 * 60 * 24);
+ }
+ } catch (e) {
+ logger.warn("Failed to set most recent success", { error: e });
+ }
+
try {
if (
process.env.USE_DB_AUTHENTICATION === "true" &&
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 313af37ff..9ea0c158f 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.1.0",
+ "version": "4.2.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts
index 7ea8506f3..e11d7527e 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/e2e/v2/usage.test.ts
@@ -32,5 +32,13 @@ describe("v2.usage e2e", () => {
const resp = await client.getTokenUsage();
expect(typeof resp.remainingTokens).toBe("number");
}, 60_000);
+
+ test("get_queue_status", async () => {
+ const resp = await client.getQueueStatus();
+ expect(typeof resp.jobsInQueue).toBe("number");
+ expect(typeof resp.activeJobsInQueue).toBe("number");
+ expect(typeof resp.waitingJobsInQueue).toBe("number");
+ expect(typeof resp.maxConcurrency).toBe("number");
+ }, 60_000);
});
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index eee0397ba..8b5b4b786 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -568,6 +568,22 @@ export interface GenerateLLMsTextStatusResponse {
expiresAt: string;
}
+/**
+ * Response interface for queue status operations.
+ */
+export interface QueueStatusResponse {
+ success: boolean;
+ jobsInQueue: number;
+ activeJobsInQueue: number;
+ waitingJobsInQueue: number;
+ maxConcurrency: number;
+
+ /**
+ * ISO timestamp of the most recent successful scrape in the past 24 hours. Will be null if no successful scrape has occurred in the past 24 hours.
+ */
+ mostRecentSuccess: string | null;
+}
+
/**
* Main class for interacting with the Firecrawl API.
* Provides methods for scraping, searching, crawling, and mapping web content.
@@ -1586,7 +1602,7 @@ export default class FirecrawlApp {
* @param {AxiosResponse} response - The response from the API.
* @param {string} action - The action being performed when the error occurred.
*/
- handleError(response: AxiosResponse, action: string): void {
+ handleError(response: AxiosResponse, action: string): never {
if (!response) {
throw new FirecrawlError(
`No response received while trying to ${action}. This may be a network error or the server is unreachable.`,
@@ -2027,6 +2043,33 @@ export default class FirecrawlApp {
}
return { success: false, error: "Internal server error." };
}
+
+ /**
+ * Gets metrics about the team's scrape queue.
+ * @returns The current queue status.
+ */
+ async getQueueStatus(): Promise<QueueStatusResponse | ErrorResponse> {
+ const headers = this.prepareHeaders();
+ try {
+ const response: AxiosResponse = await this.getRequest(
+ `${this.apiUrl}/v1/team/queue-status`,
+ headers
+ );
+
+ if (response.status === 200) {
+ return response.data;
+ } else {
+ this.handleError(response, "get queue status");
+ }
+ } catch (error: any) {
+ if (error.response?.data?.error) {
+ throw new FirecrawlError(`Request failed with status code ${error.response.status}. Error: ${error.response.data.error} ${error.response.data.details ? ` - ${JSON.stringify(error.response.data.details)}` : ''}`, error.response.status);
+ } else {
+ throw new FirecrawlError(error.message, 500);
+ }
+ }
+ return { success: false, error: "Internal server error." };
+ }
}
interface CrawlWatcherEvents {
diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts
index 0adce61ba..04f8c2b60 100644
--- a/apps/js-sdk/firecrawl/src/v2/client.ts
+++ b/apps/js-sdk/firecrawl/src/v2/client.ts
@@ -19,7 +19,7 @@ import {
batchScrape as batchWaiter,
} from "./methods/batch";
import { startExtract, getExtractStatus, extract as extractWaiter } from "./methods/extract";
-import { getConcurrency, getCreditUsage, getTokenUsage } from "./methods/usage";
+import { getConcurrency, getCreditUsage, getQueueStatus, getTokenUsage } from "./methods/usage";
import type {
Document,
ScrapeOptions,
@@ -269,6 +269,11 @@ export class FirecrawlClient {
return getTokenUsage(this.http);
}
+ /** Metrics about the team's scrape queue. */
+ async getQueueStatus() {
+ return getQueueStatus(this.http);
+ }
+
// Watcher
/**
* Create a watcher for a crawl or batch job. Emits: `document`, `snapshot`, `done`, `error`.
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/usage.ts b/apps/js-sdk/firecrawl/src/v2/methods/usage.ts
index 37c3f73e0..a3a471df8 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/usage.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/usage.ts
@@ -1,4 +1,4 @@
-import type { ConcurrencyCheck, CreditUsage, TokenUsage } from "../types";
+import type { ConcurrencyCheck, CreditUsage, QueueStatusResponse, TokenUsage } from "../types";
import { HttpClient } from "../utils/httpClient";
import { normalizeAxiosError, throwForBadResponse } from "../utils/errorHandler";
@@ -37,3 +37,13 @@ export async function getTokenUsage(http: HttpClient): Promise<TokenUsage> {
}
}
+export async function getQueueStatus(http: HttpClient): Promise<QueueStatusResponse> {
+ try {
+ const res = await http.get<QueueStatusResponse>("/v2/team/queue-status");
+ if (res.status !== 200 || !res.data?.success) throwForBadResponse(res, "get queue status");
+ return res.data;
+ } catch (err: any) {
+ if (err?.isAxiosError) return normalizeAxiosError(err, "get queue status");
+ throw err;
+ }
+}
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 1139df6f0..83bddcf4c 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -388,3 +388,11 @@ export class SdkError extends Error {
}
}
+export interface QueueStatusResponse {
+ success: boolean;
+ jobsInQueue: number;
+ activeJobsInQueue: number;
+ waitingJobsInQueue: number;
+ maxConcurrency: number;
+ mostRecentSuccess: string | null;
+}
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 35b3983dc..8c47d7995 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -17,7 +17,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.1.1"
+__version__ = "4.2.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index a4bc2f60b..c97e5089a 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -727,6 +727,10 @@ class FirecrawlClient:
"""Get recent token usage metrics (v2)."""
return usage_methods.get_token_usage(self.http_client)
+ def get_queue_status(self):
+ """Get metrics about the team's scrape queue."""
+ return usage_methods.get_queue_status(self.http_client)
+
def watcher(
self,
job_id: str,
diff --git a/apps/python-sdk/firecrawl/v2/methods/usage.py b/apps/python-sdk/firecrawl/v2/methods/usage.py
index c9888fe5c..418176780 100644
--- a/apps/python-sdk/firecrawl/v2/methods/usage.py
+++ b/apps/python-sdk/firecrawl/v2/methods/usage.py
@@ -1,5 +1,5 @@
from ..utils import HttpClient, handle_response_error
-from ..types import ConcurrencyCheck, CreditUsage, TokenUsage
+from ..types import ConcurrencyCheck, CreditUsage, QueueStatusResponse, TokenUsage
def get_concurrency(client: HttpClient) -> ConcurrencyCheck:
@@ -39,3 +39,18 @@ def get_token_usage(client: HttpClient) -> TokenUsage:
remaining_tokens=data.get("remainingTokens", 0)
)
+def get_queue_status(client: HttpClient) -> QueueStatusResponse:
+ resp = client.get("/v2/team/queue-status")
+ if not resp.ok:
+ handle_response_error(resp, "get queue status")
+ body = resp.json()
+ if not body.get("success"):
+ raise Exception(body.get("error", "Unknown error"))
+ data = body.get("data", body)
+ return QueueStatusResponse(
+ jobs_in_queue=data.get("jobsInQueue", 0),
+ active_jobs_in_queue=data.get("activeJobsInQueue", 0),
+ waiting_jobs_in_queue=data.get("waitingJobsInQueue", 0),
+ max_concurrency=data.get("maxConcurrency", 0),
+ most_recent_success=data.get("mostRecentSuccess", None),
+ )
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 2a56be41a..f3f613b35 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -485,6 +485,14 @@ class TokenUsage(BaseModel):
"""Recent token usage metrics (if available)."""
remaining_tokens: int
+class QueueStatusResponse(BaseModel):
+ """Metrics about the team's scrape queue."""
+ jobs_in_queue: int
+ active_jobs_in_queue: int
+ waiting_jobs_in_queue: int
+ max_concurrency: int
+ most_recent_success: Optional[datetime] = None
+
# Action types
class WaitAction(BaseModel):
"""Wait action to perform during scraping."""