Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): GET /crawl/ongoing (FIR-2189) (#1620)

Gergő Móricza05c4ae97df2
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.ts
index ceb66631b..6f8f1ca15 100644
--- a/apps/api/src/__tests__/snips/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/crawl.test.ts
@@ -1,4 +1,4 @@
-import { crawl } from "./lib";
+import { asyncCrawl, asyncCrawlWaitForFinish, crawl, crawlOngoing } from "./lib";
import { describe, it, expect } from "@jest/globals";
describe("Crawl tests", () => {
@@ -45,6 +45,23 @@ describe("Crawl tests", () => {
delay: 5,
});
}, 300000);
+
+ it.concurrent("ongoing crawls endpoint works", async () => {
+ const res = await asyncCrawl({
+ url: "https://firecrawl.dev",
+ limit: 3,
+ });
+
+ const ongoing = await crawlOngoing();
+
+ expect(ongoing.crawls.find(x => x.id === res.id)).toBeDefined();
+
+ await asyncCrawlWaitForFinish(res.id);
+
+ const ongoing2 = await crawlOngoing();
+
+ expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined();
+ }, 120000);
// TEMP: Flaky
// it.concurrent("discovers URLs properly when origin is not included", async () => {
diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.ts
index d851ca821..a4c71f3a9 100644
--- a/apps/api/src/__tests__/snips/lib.ts
+++ b/apps/api/src/__tests__/snips/lib.ts
@@ -1,7 +1,7 @@
import { configDotenv } from "dotenv";
configDotenv();
-import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, ErrorResponse } from "../../controllers/v1/types";
+import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, CrawlResponse, OngoingCrawlsResponse, ErrorResponse } from "../../controllers/v1/types";
import request from "supertest";
// =========================================
@@ -90,6 +90,20 @@ async function crawlStatus(id: string) {
.send();
}
+async function crawlOngoingRaw() {
+ return await request(TEST_URL)
+ .get("/v1/crawl/ongoing")
+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)
+ .send();
+}
+
+export async function crawlOngoing(): Promise<Exclude<OngoingCrawlsResponse, ErrorResponse>> {
+ const res = await crawlOngoingRaw();
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).toBe(true);
+ return res.body;
+}
+
function expectCrawlStartToSucceed(response: Awaited<ReturnType<typeof crawlStart>>) {
expect(response.statusCode).toBe(200);
expect(response.body.success).toBe(true);
@@ -106,6 +120,25 @@ function expectCrawlToSucceed(response: Awaited<ReturnType<typeof crawlStatus>>)
expect(response.body.data.length).toBeGreaterThan(0);
}
+export async function asyncCrawl(body: CrawlRequestInput): Promise<Exclude<CrawlResponse, ErrorResponse>> {
+ const cs = await crawlStart(body);
+ expectCrawlStartToSucceed(cs);
+ return cs.body;
+}
+
+export async function asyncCrawlWaitForFinish(id: string): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {
+ let x;
+
+ do {
+ x = await crawlStatus(id);
+ expect(x.statusCode).toBe(200);
+ expect(typeof x.body.status).toBe("string");
+ } while (x.body.status === "scraping");
+
+ expectCrawlToSucceed(x);
+ return x.body;
+}
+
export async function crawl(body: CrawlRequestInput): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {
const cs = await crawlStart(body);
expectCrawlStartToSucceed(cs);
diff --git a/apps/api/src/controllers/v1/crawl-ongoing.ts b/apps/api/src/controllers/v1/crawl-ongoing.ts
new file mode 100644
index 000000000..0b8ef1446
--- /dev/null
+++ b/apps/api/src/controllers/v1/crawl-ongoing.ts
@@ -0,0 +1,34 @@
+import { Response } from "express";
+import {
+ OngoingCrawlsResponse,
+ RequestWithAuth,
+ toNewCrawlerOptions,
+} from "./types";
+import {
+ getCrawl,
+ getCrawlsByTeamId,
+} from "../../lib/crawl-redis";
+import { configDotenv } from "dotenv";
+configDotenv();
+
+export async function ongoingCrawlsController(
+ req: RequestWithAuth<{}, undefined, OngoingCrawlsResponse>,
+ res: Response<OngoingCrawlsResponse>,
+) {
+ const ids = await getCrawlsByTeamId(req.auth.team_id);
+
+ const crawls = (await Promise.all(ids.map(async id => ({ ...(await getCrawl(id)), id })))).filter((crawl) => crawl !== null && !crawl.cancelled);
+
+ res.status(200).json({
+ success: true,
+ crawls: crawls.map(x => ({
+ id: x.id,
+ teamId: x.team_id!,
+ url: x.originUrl!,
+ options: {
+ ...toNewCrawlerOptions(x.crawlerOptions),
+ scrapeOptions: x.scrapeOptions,
+ },
+ })),
+ });
+}
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index 88433b4c4..c36821e48 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -882,6 +882,18 @@ export type CrawlStatusResponse =
data: Document[];
};
+export type OngoingCrawlsResponse =
+ | ErrorResponse
+ | {
+ success: true;
+ crawls: {
+ id: string;
+ teamId: string;
+ url: string;
+ options: CrawlerOptions;
+ }[];
+ };
+
export type CrawlErrorsResponse =
| ErrorResponse
| {
@@ -1016,6 +1028,25 @@ export function toLegacyCrawlerOptions(x: CrawlerOptions) {
};
}
+export function toNewCrawlerOptions(x: any): CrawlerOptions {
+ return {
+ includePaths: x.includes,
+ excludePaths: x.excludes,
+ limit: x.limit,
+ maxDepth: x.maxDepth,
+ allowBackwardLinks: x.allowBackwardCrawling,
+ allowExternalLinks: x.allowExternalContentLinks,
+ allowSubdomains: x.allowSubdomains,
+ ignoreRobotsTxt: x.ignoreRobotsTxt,
+ ignoreSitemap: x.ignoreSitemap,
+ deduplicateSimilarURLs: x.deduplicateSimilarURLs,
+ ignoreQueryParameters: x.ignoreQueryParameters,
+ regexOnFullURL: x.regexOnFullURL,
+ maxDiscoveryDepth: x.maxDiscoveryDepth,
+ delay: x.delay,
+ }
+}
+
export function fromLegacyCrawlerOptions(x: any, teamId: string): {
crawlOptions: CrawlerOptions;
internalOptions: InternalOptions;
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index bcaa76a81..710c513d4 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -26,6 +26,13 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {
});
await redisEvictConnection.set("crawl:" + id, JSON.stringify(crawl));
await redisEvictConnection.expire("crawl:" + id, 24 * 60 * 60);
+
+ await redisEvictConnection.sadd("crawls_by_team_id:" + crawl.team_id, id);
+ await redisEvictConnection.expire("crawls_by_team_id:" + crawl.team_id, 24 * 60 * 60);
+}
+
+export async function getCrawlsByTeamId(team_id: string): Promise<string[]> {
+ return await redisEvictConnection.smembers("crawls_by_team_id:" + team_id);
}
export async function getCrawl(id: string): Promise<StoredCrawl | null> {
@@ -184,6 +191,12 @@ export async function finishCrawl(id: string) {
});
await redisEvictConnection.set("crawl:" + id + ":finish", "yes");
await redisEvictConnection.expire("crawl:" + id + ":finish", 24 * 60 * 60);
+
+ const crawl = await getCrawl(id);
+ if (crawl && crawl.team_id) {
+ await redisEvictConnection.srem("crawls_by_team_id:" + crawl.team_id, id);
+ await redisEvictConnection.expire("crawls_by_team_id:" + crawl.team_id, 24 * 60 * 60);
+ }
}
export async function getCrawlJobs(id: string): Promise<string[]> {
diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.ts
index 686653bc0..61190943c 100644
--- a/apps/api/src/routes/v1.ts
+++ b/apps/api/src/routes/v1.ts
@@ -35,6 +35,7 @@ import { generateLLMsTextStatusController } from "../controllers/v1/generate-llm
import { deepResearchController } from "../controllers/v1/deep-research";
import { deepResearchStatusController } from "../controllers/v1/deep-research-status";
import { tokenUsageController } from "../controllers/v1/token-usage";
+import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";
function checkCreditsMiddleware(
minimum?: number,
@@ -213,6 +214,12 @@ v1Router.post(
wrap(mapController),
);
+v1Router.get(
+ "/crawl/ongoing",
+ authMiddleware(RateLimiterMode.CrawlStatus),
+ wrap(ongoingCrawlsController),
+);
+
v1Router.get(
"/crawl/:jobId",
authMiddleware(RateLimiterMode.CrawlStatus),