Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): GET /crawl/ongoing (FIR-2189) (#1620)
Gergő Móricza05c4ae97df2
diff --git a/apps/api/src/__tests__/snips/crawl.test.ts b/apps/api/src/__tests__/snips/crawl.test.tsindex ceb66631b..6f8f1ca15 100644--- a/apps/api/src/__tests__/snips/crawl.test.ts+++ b/apps/api/src/__tests__/snips/crawl.test.ts@@ -1,4 +1,4 @@-import { crawl } from "./lib";+import { asyncCrawl, asyncCrawlWaitForFinish, crawl, crawlOngoing } from "./lib";import { describe, it, expect } from "@jest/globals";describe("Crawl tests", () => {@@ -45,6 +45,23 @@ describe("Crawl tests", () => {delay: 5,});}, 300000);++ it.concurrent("ongoing crawls endpoint works", async () => {+ const res = await asyncCrawl({+ url: "https://firecrawl.dev",+ limit: 3,+ });++ const ongoing = await crawlOngoing();++ expect(ongoing.crawls.find(x => x.id === res.id)).toBeDefined();++ await asyncCrawlWaitForFinish(res.id);++ const ongoing2 = await crawlOngoing();++ expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined();+ }, 120000);// TEMP: Flaky// it.concurrent("discovers URLs properly when origin is not included", async () => {diff --git a/apps/api/src/__tests__/snips/lib.ts b/apps/api/src/__tests__/snips/lib.tsindex d851ca821..a4c71f3a9 100644--- a/apps/api/src/__tests__/snips/lib.ts+++ b/apps/api/src/__tests__/snips/lib.ts@@ -1,7 +1,7 @@import { configDotenv } from "dotenv";configDotenv();-import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, ErrorResponse } from "../../controllers/v1/types";+import { ScrapeRequestInput, Document, ExtractRequestInput, ExtractResponse, CrawlRequestInput, MapRequestInput, BatchScrapeRequestInput, SearchRequestInput, CrawlStatusResponse, CrawlResponse, OngoingCrawlsResponse, ErrorResponse } from "../../controllers/v1/types";import request from "supertest";// =========================================@@ -90,6 +90,20 @@ async function crawlStatus(id: string) {.send();}+async function crawlOngoingRaw() {+ return await request(TEST_URL)+ .get("/v1/crawl/ongoing")+ .set("Authorization", `Bearer ${process.env.TEST_API_KEY}`)+ .send();+}++export async function crawlOngoing(): Promise<Exclude<OngoingCrawlsResponse, ErrorResponse>> {+ const res = await crawlOngoingRaw();+ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ return res.body;+}+function expectCrawlStartToSucceed(response: Awaited<ReturnType<typeof crawlStart>>) {expect(response.statusCode).toBe(200);expect(response.body.success).toBe(true);@@ -106,6 +120,25 @@ function expectCrawlToSucceed(response: Awaited<ReturnType<typeof crawlStatus>>)expect(response.body.data.length).toBeGreaterThan(0);}+export async function asyncCrawl(body: CrawlRequestInput): Promise<Exclude<CrawlResponse, ErrorResponse>> {+ const cs = await crawlStart(body);+ expectCrawlStartToSucceed(cs);+ return cs.body;+}++export async function asyncCrawlWaitForFinish(id: string): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {+ let x;++ do {+ x = await crawlStatus(id);+ expect(x.statusCode).toBe(200);+ expect(typeof x.body.status).toBe("string");+ } while (x.body.status === "scraping");++ expectCrawlToSucceed(x);+ return x.body;+}+export async function crawl(body: CrawlRequestInput): Promise<Exclude<CrawlStatusResponse, ErrorResponse>> {const cs = await crawlStart(body);expectCrawlStartToSucceed(cs);diff --git a/apps/api/src/controllers/v1/crawl-ongoing.ts b/apps/api/src/controllers/v1/crawl-ongoing.tsnew file mode 100644index 000000000..0b8ef1446--- /dev/null+++ b/apps/api/src/controllers/v1/crawl-ongoing.ts@@ -0,0 +1,34 @@+import { Response } from "express";+import {+ OngoingCrawlsResponse,+ RequestWithAuth,+ toNewCrawlerOptions,+} from "./types";+import {+ getCrawl,+ getCrawlsByTeamId,+} from "../../lib/crawl-redis";+import { configDotenv } from "dotenv";+configDotenv();++export async function ongoingCrawlsController(+ req: RequestWithAuth<{}, undefined, OngoingCrawlsResponse>,+ res: Response<OngoingCrawlsResponse>,+) {+ const ids = await getCrawlsByTeamId(req.auth.team_id);++ const crawls = (await Promise.all(ids.map(async id => ({ ...(await getCrawl(id)), id })))).filter((crawl) => crawl !== null && !crawl.cancelled);++ res.status(200).json({+ success: true,+ crawls: crawls.map(x => ({+ id: x.id,+ teamId: x.team_id!,+ url: x.originUrl!,+ options: {+ ...toNewCrawlerOptions(x.crawlerOptions),+ scrapeOptions: x.scrapeOptions,+ },+ })),+ });+}diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 88433b4c4..c36821e48 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -882,6 +882,18 @@ export type CrawlStatusResponse =data: Document[];};+export type OngoingCrawlsResponse =+ | ErrorResponse+ | {+ success: true;+ crawls: {+ id: string;+ teamId: string;+ url: string;+ options: CrawlerOptions;+ }[];+ };+export type CrawlErrorsResponse =| ErrorResponse| {@@ -1016,6 +1028,25 @@ export function toLegacyCrawlerOptions(x: CrawlerOptions) {};}+export function toNewCrawlerOptions(x: any): CrawlerOptions {+ return {+ includePaths: x.includes,+ excludePaths: x.excludes,+ limit: x.limit,+ maxDepth: x.maxDepth,+ allowBackwardLinks: x.allowBackwardCrawling,+ allowExternalLinks: x.allowExternalContentLinks,+ allowSubdomains: x.allowSubdomains,+ ignoreRobotsTxt: x.ignoreRobotsTxt,+ ignoreSitemap: x.ignoreSitemap,+ deduplicateSimilarURLs: x.deduplicateSimilarURLs,+ ignoreQueryParameters: x.ignoreQueryParameters,+ regexOnFullURL: x.regexOnFullURL,+ maxDiscoveryDepth: x.maxDiscoveryDepth,+ delay: x.delay,+ }+}+export function fromLegacyCrawlerOptions(x: any, teamId: string): {crawlOptions: CrawlerOptions;internalOptions: InternalOptions;diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex bcaa76a81..710c513d4 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -26,6 +26,13 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {});await redisEvictConnection.set("crawl:" + id, JSON.stringify(crawl));await redisEvictConnection.expire("crawl:" + id, 24 * 60 * 60);++ await redisEvictConnection.sadd("crawls_by_team_id:" + crawl.team_id, id);+ await redisEvictConnection.expire("crawls_by_team_id:" + crawl.team_id, 24 * 60 * 60);+}++export async function getCrawlsByTeamId(team_id: string): Promise<string[]> {+ return await redisEvictConnection.smembers("crawls_by_team_id:" + team_id);}export async function getCrawl(id: string): Promise<StoredCrawl | null> {@@ -184,6 +191,12 @@ export async function finishCrawl(id: string) {});await redisEvictConnection.set("crawl:" + id + ":finish", "yes");await redisEvictConnection.expire("crawl:" + id + ":finish", 24 * 60 * 60);++ const crawl = await getCrawl(id);+ if (crawl && crawl.team_id) {+ await redisEvictConnection.srem("crawls_by_team_id:" + crawl.team_id, id);+ await redisEvictConnection.expire("crawls_by_team_id:" + crawl.team_id, 24 * 60 * 60);+ }}export async function getCrawlJobs(id: string): Promise<string[]> {diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 686653bc0..61190943c 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -35,6 +35,7 @@ import { generateLLMsTextStatusController } from "../controllers/v1/generate-llmimport { deepResearchController } from "../controllers/v1/deep-research";import { deepResearchStatusController } from "../controllers/v1/deep-research-status";import { tokenUsageController } from "../controllers/v1/token-usage";+import { ongoingCrawlsController } from "../controllers/v1/crawl-ongoing";function checkCreditsMiddleware(minimum?: number,@@ -213,6 +214,12 @@ v1Router.post(wrap(mapController),);+v1Router.get(+ "/crawl/ongoing",+ authMiddleware(RateLimiterMode.CrawlStatus),+ wrap(ongoingCrawlsController),+);+v1Router.get("/crawl/:jobId",authMiddleware(RateLimiterMode.CrawlStatus),