Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(crawl): track running crawls in Redis

Gergő Móricze50a270986cf
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index d2442d811..59c38eb89 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -20,6 +20,7 @@ import {
finishCrawlKickoff,
lockURL,
lockURLs,
+ markCrawlActive,
saveCrawl,
StoredCrawl,
} from "../../../src/lib/crawl-redis";
@@ -204,6 +205,8 @@ export async function crawlController(req: Request, res: Response) {
await saveCrawl(id, sc);
+ await markCrawlActive(id);
+
await finishCrawlKickoff(id);
const sitemap = sc.crawlerOptions.ignoreSitemap
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 256db5c1f..101821c3c 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -13,6 +13,7 @@ import {
finishCrawlKickoff,
getCrawl,
lockURLs,
+ markCrawlActive,
saveCrawl,
StoredCrawl,
} from "../../lib/crawl-redis";
@@ -129,6 +130,7 @@ export async function batchScrapeController(
if (!req.body.appendToId) {
await saveCrawl(id, sc);
+ await markCrawlActive(id);
}
let jobPriority = 20;
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index dfbd8597a..e96f36608 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -7,7 +7,12 @@ import {
RequestWithAuth,
toLegacyCrawlerOptions,
} from "./types";
-import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";
+import {
+ crawlToCrawler,
+ saveCrawl,
+ StoredCrawl,
+ markCrawlActive,
+} from "../../lib/crawl-redis";
import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
import { fromV1ScrapeOptions } from "../v2/types";
@@ -132,6 +137,8 @@ export async function crawlController(
await saveCrawl(id, sc);
+ await markCrawlActive(id);
+
await _addScrapeJobToBullMQ(
{
mode: "kickoff" as const,
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 09a6af15c..c1d15a5e1 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -14,6 +14,7 @@ import {
finishCrawlKickoff,
getCrawl,
lockURLs,
+ markCrawlActive,
saveCrawl,
StoredCrawl,
} from "../../lib/crawl-redis";
@@ -122,6 +123,7 @@ export async function batchScrapeController(
if (!req.body.appendToId) {
await saveCrawl(id, sc);
+ await markCrawlActive(id);
}
let jobPriority = 20;
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index b9affea48..b80365ade 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -7,7 +7,12 @@ import {
RequestWithAuth,
toV0CrawlerOptions,
} from "./types";
-import { crawlToCrawler, saveCrawl, StoredCrawl } from "../../lib/crawl-redis";
+import {
+ crawlToCrawler,
+ saveCrawl,
+ StoredCrawl,
+ markCrawlActive,
+} from "../../lib/crawl-redis";
import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
import { generateCrawlerOptionsFromPrompt } from "../../scraper/scrapeURL/transformers/llmExtract";
@@ -194,6 +199,8 @@ export async function crawlController(
await saveCrawl(id, sc);
+ await markCrawlActive(id);
+
await _addScrapeJobToBullMQ(
{
url: req.body.url,
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index a70b18ec9..615ae4d8d 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -49,6 +49,10 @@ export async function saveCrawl(id: string, crawl: StoredCrawl) {
});
}
+export async function markCrawlActive(id: string) {
+ await redisEvictConnection.sadd("active_crawls", id);
+}
+
export async function getCrawlsByTeamId(team_id: string): Promise<string[]> {
return await redisEvictConnection.smembers("crawls_by_team_id:" + team_id);
}
@@ -298,6 +302,8 @@ export async function finishCrawl(id: string, __logger: Logger = _logger) {
await redisEvictConnection.set("crawl:" + id + ":finish", "yes");
await redisEvictConnection.expire("crawl:" + id + ":finish", 24 * 60 * 60);
+ await redisEvictConnection.srem("active_crawls", id);
+
const crawl = await getCrawl(id);
if (crawl && crawl.team_id) {
await redisEvictConnection.srem("crawls_by_team_id:" + crawl.team_id, id);