Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawl): track running crawls in Redis
Gergő Móricze50a270986cf
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex d2442d811..59c38eb89 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -20,6 +20,7 @@ import {finishCrawlKickoff,lockURL,lockURLs,+ markCrawlActive,saveCrawl,StoredCrawl,} from "../../../src/lib/crawl-redis";@@ -204,6 +205,8 @@ export async function crawlController(req: Request, res: Response) {await saveCrawl(id, sc);+ await markCrawlActive(id);+await finishCrawlKickoff(id);const sitemap = sc.crawlerOptions.ignoreSitemapdiff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 256db5c1f..101821c3c 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -13,6 +13,7 @@ import {finishCrawlKickoff,getCrawl,lockURLs,+ markCrawlActive,saveCrawl,StoredCrawl,} from "../../lib/crawl-redis";@@ -129,6 +130,7 @@ export async function batchScrapeController(if (!req.body.appendToId) {await saveCrawl(id, sc);+ await markCrawlActive(id);}let jobPriority = 20;diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex dfbd8597a..e96f36608 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -7,7 +7,12 @@ import {RequestWithAuth,toLegacyCrawlerOptions,} from "./types";-import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";+import {+ crawlToCrawler,+ saveCrawl,+ StoredCrawl,+ markCrawlActive,+} from "../../lib/crawl-redis";import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { fromV1ScrapeOptions } from "../v2/types";@@ -132,6 +137,8 @@ export async function crawlController(await saveCrawl(id, sc);+ await markCrawlActive(id);+await _addScrapeJobToBullMQ({mode: "kickoff" as const,diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 09a6af15c..c1d15a5e1 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -14,6 +14,7 @@ import {finishCrawlKickoff,getCrawl,lockURLs,+ markCrawlActive,saveCrawl,StoredCrawl,} from "../../lib/crawl-redis";@@ -122,6 +123,7 @@ export async function batchScrapeController(if (!req.body.appendToId) {await saveCrawl(id, sc);+ await markCrawlActive(id);}let jobPriority = 20;diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex b9affea48..b80365ade 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -7,7 +7,12 @@ import {RequestWithAuth,toV0CrawlerOptions,} from "./types";-import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";+import {+ crawlToCrawler,+ saveCrawl,+ StoredCrawl,+ markCrawlActive,+} from "../../lib/crawl-redis";import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";@@ -194,6 +199,8 @@ export async function crawlController(await saveCrawl(id, sc);+ await markCrawlActive(id);+await _addScrapeJobToBullMQ({url: req.body.url,diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex a70b18ec9..615ae4d8d 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -49,6 +49,10 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {});}+export async function markCrawlActive(id: string) {+ await redisEvictConnection.sadd("active_crawls", id);+}+export async function getCrawlsByTeamId(team_id: string): Promise<string[]> {return await redisEvictConnection.smembers("crawls_by_team_id:" + team_id);}@@ -298,6 +302,8 @@ export async function finishCrawl(id: string, __logger: Logger = _logger) {await redisEvictConnection.set("crawl:" + id + ":finish", "yes");await redisEvictConnection.expire("crawl:" + id + ":finish", 24 * 60 * 60);+ await redisEvictConnection.srem("active_crawls", id);+const crawl = await getCrawl(id);if (crawl && crawl.team_id) {await redisEvictConnection.srem("crawls_by_team_id:" + crawl.team_id, id);