Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: new crawl architecture (#2320)

Gergő Móricz54f148204ff7
diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.ts
index 8bb55d43b..4be811e02 100644
--- a/apps/api/src/__tests__/snips/v1/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts
@@ -166,6 +166,9 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Crawl tests", () => {
await asyncCrawlWaitForFinish(res.id, identity);
+ // wait for crawl finish to happen on DB cron
+ await new Promise(resolve => setTimeout(resolve, 15000));
+
const ongoing2 = await crawlOngoing(identity);
expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined();
diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.ts
index 54cf66cdb..19e4bcbf6 100644
--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts
+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts
@@ -172,11 +172,14 @@ describe("Crawl tests", () => {
await asyncCrawlWaitForFinish(res.id, identity);
+ // wait for crawl finish to happen on DB cron
+ await new Promise(resolve => setTimeout(resolve, 15000));
+
const ongoing2 = await crawlOngoing(identity);
expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined();
},
- 3 * scrapeTimeout,
+ 3 * scrapeTimeout + 15000,
);
// TEMP: Flaky
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index 59c38eb89..8c795f06f 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -33,6 +33,7 @@ import { ZodError } from "zod";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { fromV0ScrapeOptions } from "../v2/types";
import { isSelfHosted } from "../../lib/deployment";
+import { crawlGroup } from "../../services/worker/nuq";
export async function crawlController(req: Request, res: Response) {
try {
@@ -203,6 +204,12 @@ export async function crawlController(req: Request, res: Response) {
sc.robots = await crawler.getRobotsTxt();
} catch (_) {}
+ await crawlGroup.addGroup(
+ id,
+ sc.team_id,
+ (chunk?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ );
+
await saveCrawl(id, sc);
await markCrawlActive(id);
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index c12ff7a9f..233e367d1 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -25,6 +25,7 @@ import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
+import { crawlGroup } from "../../services/worker/nuq";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -136,6 +137,11 @@ export async function batchScrapeController(
};
if (!req.body.appendToId) {
+ await crawlGroup.addGroup(
+ id,
+ sc.team_id,
+ (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ );
await saveCrawl(id, sc);
await markCrawlActive(id);
}
diff --git a/apps/api/src/controllers/v1/crawl-ongoing.ts b/apps/api/src/controllers/v1/crawl-ongoing.ts
index 592d31dce..74f7143be 100644
--- a/apps/api/src/controllers/v1/crawl-ongoing.ts
+++ b/apps/api/src/controllers/v1/crawl-ongoing.ts
@@ -4,15 +4,18 @@ import {
RequestWithAuth,
toNewCrawlerOptions,
} from "./types";
-import { getCrawl, getCrawlsByTeamId } from "../../lib/crawl-redis";
+import { getCrawl } from "../../lib/crawl-redis";
import { configDotenv } from "dotenv";
+import { crawlGroup } from "../../services/worker/nuq";
configDotenv();
export async function ongoingCrawlsController(
req: RequestWithAuth<{}, undefined, OngoingCrawlsResponse>,
res: Response<OngoingCrawlsResponse>,
) {
- const ids = await getCrawlsByTeamId(req.auth.team_id);
+ const ids = (await crawlGroup.getOngoingByOwner(req.auth.team_id)).map(
+ x => x.id,
+ );
const crawls = (
await Promise.all(ids.map(async id => ({ ...(await getCrawl(id)), id })))
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 673d9d315..15ed8ac7c 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -21,7 +21,12 @@ import { configDotenv } from "dotenv";
import { logger } from "../../lib/logger";
import { supabase_rr_service, supabase_service } from "../../services/supabase";
import { getJobFromGCS } from "../../lib/gcs-jobs";
-import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";
+import {
+ scrapeQueue,
+ NuQJob,
+ NuQJobStatus,
+ crawlGroup,
+} from "../../services/worker/nuq";
import { ScrapeJobSingleUrls } from "../../types";
configDotenv();
@@ -165,291 +170,109 @@ export async function crawlStatusController(
? start + parseInt(req.query.limit, 10) - 1
: undefined;
+ const group = await crawlGroup.getGroup(req.params.jobId);
+ const groupAnyJob = await scrapeQueue.getGroupAnyJob(
+ req.params.jobId,
+ req.auth.team_id,
+ );
const sc = await getCrawl(req.params.jobId);
- const djoCutoff = Date.now() - 250;
+ if (!group || (!groupAnyJob && (!sc || sc.team_id !== req.auth.team_id))) {
+ return res.status(404).json({ success: false, error: "Job not found" });
+ }
- if (sc) {
- if (sc.team_id !== req.auth.team_id) {
- return res.status(404).json({ success: false, error: "Job not found" });
- }
- } else if (process.env.USE_DB_AUTHENTICATION === "true") {
- const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("job_id", req.params.jobId)
- .limit(1);
-
- let crawlAdded: Date;
- let crawlTeamId: string;
-
- if (crawlJobError || !crawlJobs || crawlJobs.length === 0) {
- const { data: scrapeJobs, error: scrapeJobError } =
- await supabase_rr_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("crawl_id", req.params.jobId)
- .order("date_added", { ascending: false })
- .limit(1);
-
- if (scrapeJobError || !scrapeJobs || scrapeJobs.length === 0) {
- return res.status(404).json({ success: false, error: "Job not found" });
- }
-
- const scrapeJob = scrapeJobs[0];
- crawlAdded = new Date(scrapeJob.date_added); // use last scrape as date added for crawl
- crawlTeamId = scrapeJob.team_id;
- } else {
- const crawlJob = crawlJobs[0];
- crawlAdded = new Date(crawlJob.date_added);
- crawlTeamId = crawlJob.team_id;
- }
+ const zeroDataRetention = !!(
+ groupAnyJob?.data?.zeroDataRetention ?? sc?.zeroDataRetention
+ );
- if (crawlTeamId !== req.auth.team_id) {
- return res.status(404).json({ success: false, error: "Job not found" });
- }
+ const numericStats = await scrapeQueue.getGroupNumericStats(
+ req.params.jobId,
+ logger.child({ zeroDataRetention }),
+ );
- const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;
- const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;
-
- if (new Date().valueOf() - crawlAdded.valueOf() > crawlTtlMs) {
- return res.status(404).json({ success: false, error: "Job expired" });
- }
- } else {
- // if SC is gone and no DB, that means the job is expired, or never existed to begin with
- return res.status(404).json({ success: false, error: "Job not found" });
- }
+ const creditsRpc =
+ process.env.USE_DB_AUTHENTICATION === "true"
+ ? await supabase_service.rpc(
+ "credits_billed_by_crawl_id_1",
+ {
+ i_crawl_id: req.params.jobId,
+ },
+ { get: true },
+ )
+ : null;
let outputBulkA: {
status?: "completed" | "scraping" | "cancelled";
completed?: number;
total?: number;
creditsUsed?: number;
+ } = {
+ status: group.status === "active" ? "scraping" : group.status,
+ completed: numericStats.completed ?? 0,
+ total:
+ (numericStats.completed ?? 0) +
+ (numericStats.active ?? 0) +
+ (numericStats.queued ?? 0) +
+ (numericStats.backlog ?? 0),
+ creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,
};
- if (sc) {
- // Local mode (easier, low-pressure)
- const kickoffFinished = await isCrawlKickoffFinished(req.params.jobId);
- let total = await getCrawlQualifiedJobCount(req.params.jobId);
-
- let completed = await getDoneJobsOrderedLength(req.params.jobId, djoCutoff);
- let creditsUsed =
- completed *
- (sc.scrapeOptions?.formats?.find(
- x => typeof x === "object" && x.type === "json",
- )
- ? 5
- : 1);
-
- if (process.env.USE_DB_AUTHENTICATION === "true") {
- const creditsRpc = await supabase_service.rpc(
- "credits_billed_by_crawl_id_1",
- {
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- );
-
- creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? creditsUsed;
-
- if (
- total === 0 &&
- completed === 0 &&
- Date.now() - sc.createdAt > 1000 * 60
- ) {
- const x = await supabase_service.rpc(
- "crawl_status_job_count_1",
- {
- i_team_id: req.auth.team_id,
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- );
-
- total = x.data?.[0]?.count ?? 0;
- completed = total;
- }
- }
-
- outputBulkA = {
- status: sc.cancelled
- ? "cancelled"
- : completed === total && kickoffFinished
- ? "completed"
- : "scraping",
- total,
- completed,
- creditsUsed,
- };
- } else {
- // DB must be specified at this point, otherwise control flow kills execution earlier
- // DB mode (once job expires from Redis)
- const { data: crawlJobCounts, error: crawlJobCountsError } =
- await supabase_service.rpc(
- "crawl_status_job_count_1",
- {
- i_team_id: req.auth.team_id,
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- );
-
- if (crawlJobCountsError || !crawlJobCounts || crawlJobCounts.length === 0) {
- logger.error("Error getting crawl job count", {
- error: crawlJobCountsError,
- });
- throw new Error("Error getting crawl job count", {
- cause: crawlJobCountsError,
- });
- }
-
- const crawlJobCount: number = crawlJobCounts[0].count ?? 0;
-
- const creditsRpc = await supabase_service.rpc(
- "credits_billed_by_crawl_id_1",
- {
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- );
-
- const creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? crawlJobCount;
-
- outputBulkA = {
- status: "completed", // Salvage expired job
- total: crawlJobCount,
- completed: crawlJobCount,
- creditsUsed,
- };
- }
-
let outputBulkB: {
data: Document[];
next: string | undefined;
};
- if (sc || process.env.USE_DB_AUTHENTICATION !== "true") {
- // old BullMQ-based path
- const doneJobs = await getDoneJobsOrderedUntil(
- req.params.jobId,
- djoCutoff,
- start,
- end !== undefined ? end - start : 100,
- );
-
- let scrapes: Document[] = [];
- let iteratedOver = 0;
- let bytes = 0;
- const bytesLimit = 10485760; // 10 MiB in bytes
-
- for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) {
- const jobIds = doneJobs.slice(i * 50, (i + 1) * 50);
- const jobs = await getJobs(jobIds);
-
- for (const job of jobs) {
- if (job.status === "failed") {
- continue;
- } else {
- if (job?.returnvalue) {
- scrapes.push(job.returnvalue);
- bytes += JSON.stringify(job.returnvalue).length;
- } else {
- logger.warn(
- "Job was considered done, but returnvalue is undefined!",
- {
- scrapeId: job.id,
- crawlId: req.params.jobId,
- state: job.status,
- returnvalue: job?.returnvalue,
- },
- );
- }
-
- iteratedOver++;
- }
-
- if (bytes > bytesLimit) {
- break;
- }
- }
-
- if (bytes > bytesLimit) {
- break;
- }
- }
-
- if (bytes > bytesLimit && scrapes.length !== 1) {
- scrapes.splice(scrapes.length - 1, 1);
- iteratedOver--;
- }
-
- outputBulkB = {
- data: scrapes,
- next:
- (outputBulkA.total ?? 0) > start + iteratedOver
- ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
- : undefined,
- };
- } else {
- // new DB-based path
- const { data, error } = await supabase_service.rpc(
- "crawl_status_1",
- {
- i_team_id: req.auth.team_id,
- i_crawl_id: req.params.jobId,
- i_start: start,
- i_end: end ?? start + 100,
- },
- { get: true },
- );
-
- if (error || !data) {
- logger.error("Error getting crawl status from DB", { error });
- throw new Error("Error getting crawl status from DB", { cause: error });
+ const doneJobs = await scrapeQueue.getCrawlJobsForListing(
+ req.params.jobId,
+ end !== undefined ? end - start : 100,
+ start,
+ logger.child({ zeroDataRetention }),
+ );
+
+ let scrapes: Document[] = [];
+ let iteratedOver = 0;
+ let bytes = 0;
+ const bytesLimit = 10485760; // 10 MiB in bytes
+
+ const scrapeBlobs = await Promise.all(
+ doneJobs.map(
+ async x =>
+ [x.id, x.returnvalue ?? (await getJobFromGCS(x.id))?.[0]] as const,
+ ),
+ );
+
+ for (const [id, scrape] of scrapeBlobs) {
+ if (scrape) {
+ scrapes.push(scrape);
+ bytes += JSON.stringify(scrape).length;
+ } else {
+ logger.warn("Job was considered done, but returnvalue is undefined!", {
+ jobId: id,
+ returnvalue: scrape,
+ zeroDataRetention,
+ });
}
- const scrapeIds = data?.map(x => x.id) ?? [];
- let scrapes: Document[] = [];
- let iteratedOver = 0;
- let bytes = 0;
- const bytesLimit = 10485760; // 10 MiB in bytes
-
- const scrapeBlobs = await Promise.all(
- scrapeIds.map(async x => [x, (await getJobFromGCS(x))?.[0]]),
- );
-
- for (const [id, scrape] of scrapeBlobs) {
- if (scrape) {
- scrapes.push(scrape);
- bytes += JSON.stringify(scrape).length;
- } else {
- logger.warn("Job was considered done, but returnvalue is undefined!", {
- jobId: id,
- returnvalue: scrape,
- });
- }
-
- iteratedOver++;
-
- if (bytes > bytesLimit) {
- break;
- }
- }
+ iteratedOver++;
- if (bytes > bytesLimit && scrapes.length !== 1) {
- scrapes.splice(scrapes.length - 1, 1);
- iteratedOver--;
+ if (bytes > bytesLimit) {
+ break;
}
+ }
- outputBulkB = {
- data: scrapes,
- next:
- (outputBulkA.total ?? 0) > start + iteratedOver
- ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
- : undefined,
- };
+ if (bytes > bytesLimit && scrapes.length !== 1) {
+ scrapes.splice(scrapes.length - 1, 1);
+ iteratedOver--;
}
+ outputBulkB = {
+ data: scrapes,
+ next:
+ (outputBulkA.total ?? 0) > start + iteratedOver
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ : undefined,
+ };
+
return res.status(200).json({
success: true,
status: outputBulkA.status ?? "scraping",
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index e83489826..5b5fdfa7a 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -17,6 +17,7 @@ import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
import { fromV1ScrapeOptions } from "../v2/types";
import { checkPermissions } from "../../lib/permissions";
+import { crawlGroup } from "../../services/worker/nuq";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -135,6 +136,12 @@ export async function crawlController(
});
}
+ await crawlGroup.addGroup(
+ id,
+ sc.team_id,
+ (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ );
+
await saveCrawl(id, sc);
await markCrawlActive(id);
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 04a478bb1..418de7b7a 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -25,6 +25,7 @@ import { logger as _logger } from "../../lib/logger";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { checkPermissions } from "../../lib/permissions";
+import { crawlGroup } from "../../services/worker/nuq";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -129,6 +130,11 @@ export async function batchScrapeController(
};
if (!req.body.appendToId) {
+ await crawlGroup.addGroup(
+ id,
+ sc.team_id,
+ (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ );
await saveCrawl(id, sc);
await markCrawlActive(id);
}
diff --git a/apps/api/src/controllers/v2/crawl-ongoing.ts b/apps/api/src/controllers/v2/crawl-ongoing.ts
index 74d8ae84e..a27771da3 100644
--- a/apps/api/src/controllers/v2/crawl-ongoing.ts
+++ b/apps/api/src/controllers/v2/crawl-ongoing.ts
@@ -4,15 +4,18 @@ import {
RequestWithAuth,
toV2CrawlerOptions,
} from "./types";
-import { getCrawl, getCrawlsByTeamId } from "../../lib/crawl-redis";
+import { getCrawl } from "../../lib/crawl-redis";
import { configDotenv } from "dotenv";
+import { crawlGroup } from "../../services/worker/nuq";
configDotenv();
export async function ongoingCrawlsController(
req: RequestWithAuth<{}, undefined, OngoingCrawlsResponse>,
res: Response<OngoingCrawlsResponse>,
) {
- const ids = await getCrawlsByTeamId(req.auth.team_id);
+ const ids = (await crawlGroup.getOngoingByOwner(req.auth.team_id)).map(
+ x => x.id,
+ );
const crawls = (
await Promise.all(ids.map(async id => ({ ...(await getCrawl(id)), id })))
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 16e6d845c..551cf9095 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -21,7 +21,12 @@ import { configDotenv } from "dotenv";
import { logger } from "../../lib/logger";
import { supabase_rr_service, supabase_service } from "../../services/supabase";
import { getJobFromGCS } from "../../lib/gcs-jobs";
-import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";
+import {
+ scrapeQueue,
+ NuQJob,
+ NuQJobStatus,
+ crawlGroup,
+} from "../../services/worker/nuq";
import { ScrapeJobSingleUrls } from "../../types";
import { redisEvictConnection } from "../../../src/services/redis";
import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
@@ -177,298 +182,109 @@ export async function crawlStatusController(
? start + parseInt(req.query.limit, 10) - 1
: undefined;
+ const group = await crawlGroup.getGroup(req.params.jobId);
+ const groupAnyJob = await scrapeQueue.getGroupAnyJob(
+ req.params.jobId,
+ req.auth.team_id,
+ );
const sc = await getCrawl(req.params.jobId);
- const djoCutoff = Date.now() - 250;
-
- if (sc) {
- if (sc.team_id !== req.auth.team_id) {
- // Allow preview tokens to access preview jobs regardless of IP-derived team_id mismatch
- const scIsPreview = sc.team_id?.startsWith("preview");
- if (!(isPreviewTeam && scIsPreview)) {
- return res.status(404).json({ success: false, error: "Job not found" });
- }
- }
- } else if (process.env.USE_DB_AUTHENTICATION === "true") {
- if (isPreviewTeam) {
- // Preview teams do not persist crawls in DB; avoid DB lookup that can 500
- return res.status(404).json({ success: false, error: "Job not found" });
- }
- const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("job_id", req.params.jobId)
- .limit(1);
-
- let crawlAdded: Date;
- let crawlTeamId: string;
-
- if (crawlJobError || !crawlJobs || crawlJobs.length === 0) {
- const { data: scrapeJobs, error: scrapeJobError } =
- await supabase_rr_service
- .from("firecrawl_jobs")
- .select("*")
- .eq("crawl_id", req.params.jobId)
- .order("date_added", { ascending: false })
- .limit(1);
-
- if (scrapeJobError || !scrapeJobs || scrapeJobs.length === 0) {
- return res.status(404).json({ success: false, error: "Job not found" });
- }
-
- const scrapeJob = scrapeJobs[0];
- crawlAdded = new Date(scrapeJob.date_added); // use last scrape as date added for crawl
- crawlTeamId = scrapeJob.team_id;
- } else {
- const crawlJob = crawlJobs[0];
- crawlAdded = new Date(crawlJob.date_added);
- crawlTeamId = crawlJob.team_id;
- }
+ if (!group || (!groupAnyJob && (!sc || sc.team_id !== req.auth.team_id))) {
+ return res.status(404).json({ success: false, error: "Job not found" });
+ }
- if (crawlTeamId !== req.auth.team_id) {
- return res.status(404).json({ success: false, error: "Job not found" });
- }
+ const zeroDataRetention = !!(
+ groupAnyJob?.data?.zeroDataRetention ?? sc?.zeroDataRetention
+ );
- const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;
- const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;
+ const numericStats = await scrapeQueue.getGroupNumericStats(
+ req.params.jobId,
+ logger.child({ zeroDataRetention }),
+ );
- if (new Date().valueOf() - crawlAdded.valueOf() > crawlTtlMs) {
- return res.status(404).json({ success: false, error: "Job expired" });
- }
- } else {
- // if SC is gone and no DB, that means the job is expired, or never existed to begin with
- return res.status(404).json({ success: false, error: "Job not found" });
- }
+ const creditsRpc =
+ process.env.USE_DB_AUTHENTICATION === "true"
+ ? await supabase_service.rpc(
+ "credits_billed_by_crawl_id_1",
+ {
+ i_crawl_id: req.params.jobId,
+ },
+ { get: true },
+ )
+ : null;
let outputBulkA: {
status?: "completed" | "scraping" | "cancelled";
completed?: number;
total?: number;
creditsUsed?: number;
+ } = {
+ status: group.status === "active" ? "scraping" : group.status,
+ completed: numericStats.completed ?? 0,
+ total:
+ (numericStats.completed ?? 0) +
+ (numericStats.active ?? 0) +
+ (numericStats.queued ?? 0) +
+ (numericStats.backlog ?? 0),
+ creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,
};
- if (sc) {
- // Local mode (easier, low-pressure)
- const kickoffFinished = await isCrawlKickoffFinished(req.params.jobId);
- let total = await getCrawlQualifiedJobCount(req.params.jobId);
-
- let completed = await getDoneJobsOrderedLength(req.params.jobId, djoCutoff);
- let creditsUsed =
- completed *
- (sc.scrapeOptions?.formats?.find(
- x => typeof x === "object" && x.type === "json",
- )
- ? 5
- : 1);
-
- if (process.env.USE_DB_AUTHENTICATION === "true" && !isPreviewTeam) {
- const creditsRpc = await supabase_service.rpc(
- "credits_billed_by_crawl_id_1",
- {
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- );
-
- creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? creditsUsed;
-
- if (
- total === 0 &&
- completed === 0 &&
- Date.now() - sc.createdAt > 1000 * 60
- ) {
- const x = await supabase_service.rpc(
- "crawl_status_job_count_1",
- {
- i_team_id: req.auth.team_id,
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- );
-
- total = x.data?.[0]?.count ?? 0;
- completed = total;
- }
- }
-
- outputBulkA = {
- status: sc.cancelled
- ? "cancelled"
- : completed === total && kickoffFinished
- ? "completed"
- : "scraping",
- total,
- completed,
- creditsUsed,
- };
- } else {
- // DB must be specified at this point, otherwise control flow kills execution earlier
- // DB mode (once job expires from Redis)
- const { data: crawlJobCounts, error: crawlJobCountsError } =
- await supabase_service.rpc(
- "crawl_status_job_count_1",
- {
- i_team_id: req.auth.team_id,
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- );
-
- if (crawlJobCountsError || !crawlJobCounts || crawlJobCounts.length === 0) {
- logger.error("Error getting crawl job count", {
- error: crawlJobCountsError,
- });
- throw new Error("Error getting crawl job count", {
- cause: crawlJobCountsError,
- });
- }
-
- const crawlJobCount: number = crawlJobCounts[0].count ?? 0;
-
- const creditsRpc = await supabase_service.rpc(
- "credits_billed_by_crawl_id_1",
- {
- i_crawl_id: req.params.jobId,
- },
- { get: true },
- );
-
- const creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? crawlJobCount;
-
- outputBulkA = {
- status: "completed", // Salvage expired job
- total: crawlJobCount,
- completed: crawlJobCount,
- creditsUsed,
- };
- }
-
let outputBulkB: {
data: Document[];
next: string | undefined;
};
- if (sc || process.env.USE_DB_AUTHENTICATION !== "true" || isPreviewTeam) {
- const doneJobs = await getDoneJobsOrderedUntil(
- req.params.jobId,
- djoCutoff,
- start,
- end !== undefined ? end - start : 100,
- );
-
- let scrapes: Document[] = [];
- let iteratedOver = 0;
- let bytes = 0;
- const bytesLimit = 10485760; // 10 MiB in bytes
-
- for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) {
- const jobIds = doneJobs.slice(i * 50, (i + 1) * 50);
- const jobs = await getJobs(jobIds, logger);
-
- for (const job of jobs) {
- if (job.status === "failed") {
- continue;
- } else {
- if (job?.returnvalue) {
- scrapes.push(job.returnvalue);
- bytes += JSON.stringify(job.returnvalue).length;
- } else {
- logger.warn(
- "Job was considered done, but returnvalue is undefined!",
- {
- scrapeId: job.id,
- crawlId: req.params.jobId,
- state: job.status,
- returnvalue: job?.returnvalue,
- },
- );
- }
-
- iteratedOver++;
- }
-
- if (bytes > bytesLimit) {
- break;
- }
- }
-
- if (bytes > bytesLimit) {
- break;
- }
- }
-
- if (bytes > bytesLimit && scrapes.length !== 1) {
- scrapes.splice(scrapes.length - 1, 1);
- iteratedOver--;
- }
-
- outputBulkB = {
- data: scrapes,
- next:
- (outputBulkA.total ?? 0) > start + iteratedOver
- ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
- : undefined,
- };
- } else {
- // new DB-based path
- const { data, error } = await supabase_service.rpc(
- "crawl_status_1",
- {
- i_team_id: req.auth.team_id,
- i_crawl_id: req.params.jobId,
- i_start: start,
- i_end: end ?? start + 100,
- },
- { get: true },
- );
-
- if (error || !data) {
- logger.error("Error getting crawl status from DB", { error });
- throw new Error("Error getting crawl status from DB", { cause: error });
+ const doneJobs = await scrapeQueue.getCrawlJobsForListing(
+ req.params.jobId,
+ end !== undefined ? end - start : 100,
+ start,
+ logger.child({ zeroDataRetention }),
+ );
+
+ let scrapes: Document[] = [];
+ let iteratedOver = 0;
+ let bytes = 0;
+ const bytesLimit = 10485760; // 10 MiB in bytes
+
+ const scrapeBlobs = await Promise.all(
+ doneJobs.map(
+ async x =>
+ [x.id, x.returnvalue ?? (await getJobFromGCS(x.id))?.[0]] as const,
+ ),
+ );
+
+ for (const [id, scrape] of scrapeBlobs) {
+ if (scrape) {
+ scrapes.push(scrape);
+ bytes += JSON.stringify(scrape).length;
+ } else {
+ logger.warn("Job was considered done, but returnvalue is undefined!", {
+ jobId: id,
+ returnvalue: scrape,
+ zeroDataRetention,
+ });
}
- const scrapeIds = data?.map(x => x.id) ?? [];
- let scrapes: Document[] = [];
- let iteratedOver = 0;
- let bytes = 0;
- const bytesLimit = 10485760; // 10 MiB in bytes
-
- const scrapeBlobs = await Promise.all(
- scrapeIds.map(async x => [x, (await getJobFromGCS(x))?.[0]]),
- );
-
- for (const [id, scrape] of scrapeBlobs) {
- if (scrape) {
- scrapes.push(scrape);
- bytes += JSON.stringify(scrape).length;
- } else {
- logger.warn("Job was considered done, but returnvalue is undefined!", {
- jobId: id,
- returnvalue: scrape,
- });
- }
-
- iteratedOver++;
+ iteratedOver++;
- if (bytes > bytesLimit) {
- break;
- }
- }
-
- if (bytes > bytesLimit && scrapes.length !== 1) {
- scrapes.splice(scrapes.length - 1, 1);
- iteratedOver--;
+ if (bytes > bytesLimit) {
+ break;
}
+ }
- outputBulkB = {
- data: scrapes,
- next:
- (outputBulkA.total ?? 0) > start + iteratedOver
- ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
- : undefined,
- };
+ if (bytes > bytesLimit && scrapes.length !== 1) {
+ scrapes.splice(scrapes.length - 1, 1);
+ iteratedOver--;
}
+ outputBulkB = {
+ data: scrapes,
+ next:
+ (outputBulkA.total ?? 0) > start + iteratedOver
+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`
+ : undefined,
+ };
+
// Check for robots.txt blocked URLs and add warning if found
let warning: string | undefined;
try {
@@ -484,11 +300,15 @@ export async function crawlStatusController(
}
} catch (error) {
// If we can't check robots blocked URLs, continue without warning
- logger.debug("Failed to check robots blocked URLs", { error });
+ logger.debug("Failed to check robots blocked URLs", {
+ error,
+ zeroDataRetention,
+ });
}
// Check if we should warn about base domain for crawl results
- const resultCount = outputBulkA.completed ?? outputBulkA.total ?? outputBulkB.data.length;
+ const resultCount =
+ outputBulkA.completed ?? outputBulkA.total ?? outputBulkB.data.length;
const currentStatus = outputBulkA.status ?? "scraping";
if (!warning && currentStatus !== "scraping" && resultCount <= 1) {
// Get the original crawl URL and options from stored crawl data
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 2a74b00fa..d588d3a00 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -20,6 +20,7 @@ import { CostTracking } from "../../lib/cost-tracking";
import { checkPermissions } from "../../lib/permissions";
import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";
import { modifyCrawlUrl } from "../../utils/url-utils";
+import { crawlGroup } from "../../services/worker/nuq";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -202,6 +203,12 @@ export async function crawlController(
});
}
+ await crawlGroup.addGroup(
+ id,
+ sc.team_id,
+ (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ );
+
await saveCrawl(id, sc);
await markCrawlActive(id);
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 122836720..513a1ea73 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -1,7 +1,7 @@
import "dotenv/config";
import { type ChildProcess, spawn } from "child_process";
import * as net from "net";
-import { basename } from "path";
+import { basename, join } from "path";
import { HTML_TO_MARKDOWN_PATH } from "./natives";
import { createWriteStream } from "fs";
@@ -11,6 +11,15 @@ const stopping = new WeakSet<ChildProcess>(); // processes we're intentionally s
let IS_DEV = false;
let restartSignal: AbortController | null = null;
let shuttingDown = false;
+let nuqPostgresContainer: {
+ containerName: string;
+ containerRuntime: string;
+} | null = null;
+
+// Get the monorepo root (apps/api/dist/src -> ../../../..)
+// __dirname is available in CommonJS (which this compiles to)
+const MONOREPO_ROOT = join(__dirname, "..", "..", "..", "..");
+const NUQ_POSTGRES_PATH = join(MONOREPO_ROOT, "apps", "nuq-postgres");
interface ProcessResult {
promise: Promise<void>;
@@ -25,6 +34,10 @@ interface Services {
extractWorker?: ProcessResult;
indexWorker?: ProcessResult;
command?: ProcessResult;
+ nuqPostgres?: {
+ containerName: string;
+ containerRuntime: string;
+ };
}
const colors = {
@@ -49,6 +62,8 @@ const processGroupColors: Record<string, string> = {
index: colors.yellow,
go: colors.yellow,
command: colors.white,
+ docker: colors.blue,
+ podman: colors.blue,
};
function getProcessGroup(name: string): string {
@@ -345,6 +360,174 @@ function terminateProcess(proc: ChildProcess, force: boolean): Promise<void> {
});
}
+async function detectContainerRuntime(): Promise<string | null> {
+ // Try docker first, then podman
+ for (const runtime of ["docker", "podman"]) {
+ try {
+ const check = execForward(`${runtime}@check`, `${runtime} --version`);
+ await check.promise;
+ return runtime;
+ } catch {
+ // Runtime not available, try next
+ }
+ }
+ return null;
+}
+
+async function isContainerRunning(
+ runtime: string,
+ containerName: string,
+): Promise<boolean> {
+ try {
+ const check = execForward(
+ `${runtime}@ps`,
+ `${runtime} ps -a --filter name=^${containerName}$ --format '{{.Names}}'`,
+ );
+ await check.promise;
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+async function stopAndRemoveContainer(
+ runtime: string,
+ containerName: string,
+): Promise<void> {
+ const isRunning = await isContainerRunning(runtime, containerName);
+ if (isRunning) {
+ logger.info(`Stopping existing container: ${containerName}`);
+ try {
+ const stop = execForward(
+ `${runtime}@stop`,
+ `${runtime} stop ${containerName}`,
+ );
+ await stop.promise;
+ } catch (e) {
+ logger.warn(`Failed to stop container ${containerName}, continuing...`);
+ }
+ }
+
+ // Try to remove the container (whether it was running or not)
+ try {
+ const remove = execForward(
+ `${runtime}@rm`,
+ `${runtime} rm -f ${containerName}`,
+ );
+ await remove.promise;
+ } catch (e) {
+ // Container might not exist, that's fine
+ }
+}
+
+async function buildNuqPostgresImage(runtime: string): Promise<void> {
+ logger.info("Building nuq-postgres Docker image");
+ const build = execForward(
+ `${runtime}@build`,
+ `${runtime} build -t firecrawl-nuq-postgres:latest ${NUQ_POSTGRES_PATH}`,
+ );
+ await build.promise;
+ logger.success("nuq-postgres image built");
+}
+
+async function startNuqPostgresContainer(
+ runtime: string,
+ containerName: string,
+): Promise<void> {
+ logger.info(`Starting PostgreSQL container: ${containerName}`);
+ const start = execForward(
+ `${runtime}@start`,
+ `${runtime} run -d --name ${containerName} -p 5432:5432 -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres -e POSTGRES_DB=postgres firecrawl-nuq-postgres:latest`,
+ );
+ await start.promise;
+ logger.success(`PostgreSQL container started: ${containerName}`);
+}
+
+async function waitForPostgres(
+ host: string,
+ port: number,
+ timeoutMs: number = 30000,
+): Promise<void> {
+ logger.info("Waiting for PostgreSQL to be ready...");
+ const start = Date.now();
+
+ while (Date.now() - start < timeoutMs) {
+ try {
+ // Try to connect to PostgreSQL using a simple query
+ const { Client } = await import("pg");
+ const client = new Client({
+ host,
+ port,
+ user: "postgres",
+ password: "postgres",
+ database: "postgres",
+ connectionTimeoutMillis: 2000,
+ });
+
+ await client.connect();
+ await client.query("SELECT 1");
+ await client.end();
+
+ logger.success("PostgreSQL is ready");
+ return;
+ } catch (e) {
+ // Not ready yet, wait and retry
+ await new Promise(resolve => setTimeout(resolve, 500));
+ }
+ }
+
+ throw new Error(`PostgreSQL did not become ready within ${timeoutMs}ms`);
+}
+
+async function setupNuqPostgres(): Promise<Services["nuqPostgres"]> {
+ if (process.env.NUQ_DATABASE_URL) {
+ logger.info("NUQ_DATABASE_URL is set, skipping container management");
+ return undefined;
+ }
+
+ logger.section("Setting up NUQ PostgreSQL container");
+
+ const runtime = await detectContainerRuntime();
+ if (!runtime) {
+ throw new Error(
+ "Neither Docker nor Podman found. Please install Docker/Podman or set NUQ_DATABASE_URL manually.",
+ );
+ }
+
+ logger.success(`Using container runtime: ${runtime}`);
+
+ const containerName = "firecrawl-nuq-postgres";
+
+ // Stop and remove any existing container
+ await stopAndRemoveContainer(runtime, containerName);
+
+ // Build the image
+ await buildNuqPostgresImage(runtime);
+
+ // Start the container
+ await startNuqPostgresContainer(runtime, containerName);
+
+ // Wait for PostgreSQL to be ready
+ await waitForPostgres("localhost", 5432);
+
+ // Set environment variables for the services
+ const dbUrl = "postgresql://postgres:postgres@localhost:5432/postgres";
+ process.env.NUQ_DATABASE_URL = dbUrl;
+ process.env.NUQ_DATABASE_URL_LISTEN = dbUrl;
+
+ logger.success("NUQ PostgreSQL container is ready");
+
+ const containerInfo = {
+ containerName,
+ containerRuntime: runtime,
+ };
+
+ // Store globally for graceful shutdown
+ nuqPostgresContainer = containerInfo;
+
+ return containerInfo;
+}
+
async function installDependencies() {
logger.section("Installing dependencies");
@@ -388,6 +571,9 @@ async function installDependencies() {
}
async function startServices(command?: string[]): Promise<Services> {
+ // Setup NUQ PostgreSQL container if needed
+ const nuqPostgres = await setupNuqPostgres();
+
logger.section("Starting services");
const api = execForward(
@@ -492,6 +678,7 @@ async function startServices(command?: string[]): Promise<Services> {
indexWorker,
extractWorker,
command: commandProcess,
+ nuqPostgres,
};
}
@@ -515,6 +702,16 @@ async function stopDevelopmentServices(services: Services) {
logger.error("Error while stopping processes", e);
}),
]);
+
+ // Stop and remove NUQ PostgreSQL container if it was started by harness
+ if (services.nuqPostgres) {
+ logger.info("Stopping NUQ PostgreSQL container");
+ await stopAndRemoveContainer(
+ services.nuqPostgres.containerRuntime,
+ services.nuqPostgres.containerName,
+ );
+ logger.success("NUQ PostgreSQL container stopped");
+ }
}
async function runDevMode(): Promise<void> {
@@ -664,6 +861,18 @@ async function gracefulShutdown() {
terminateProcess(proc, forceTerminate),
);
await Promise.allSettled(terminationPromises);
+
+ // Stop and remove NUQ PostgreSQL container if it was started by harness
+ if (nuqPostgresContainer) {
+ logger.info("Stopping NUQ PostgreSQL container");
+ await stopAndRemoveContainer(
+ nuqPostgresContainer.containerRuntime,
+ nuqPostgresContainer.containerName,
+ );
+ logger.success("NUQ PostgreSQL container stopped");
+ nuqPostgresContainer = null;
+ }
+
logger.success("All processes terminated");
}
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index c0a435478..03a6ccdaf 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -20,6 +20,10 @@ import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { BullMQOtel } from "bullmq-otel";
import { getErrorContactMessage } from "../lib/deployment";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
+import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";
+import { getCrawl } from "../lib/crawl-redis";
+import type { Logger } from "winston";
+import { finishCrawlSuper } from "./worker/crawl-logic";
configDotenv();
@@ -249,6 +253,117 @@ const workerFun = async (
}
};
+async function processFinishCrawlJobInternal(job: NuQJob, logger: Logger) {
+ if (!job.groupId) {
+ throw new Error("crawlFinish job with no groupId");
+ }
+
+ if (!job.ownerId) {
+ throw new Error("crawlFinish job with no ownerId");
+ }
+
+ const sc = await getCrawl(job.groupId);
+
+ if (!sc) {
+ throw new Error("crawlFinish job with sc expired");
+ }
+
+ const anyJob = await scrapeQueue.getGroupAnyJob(job.groupId, job.ownerId);
+
+ if (!anyJob) {
+ throw new Error("crawlFinish couldn't find anyJob");
+ }
+
+ await finishCrawlSuper(anyJob);
+}
+
+const crawlFinishWorker = async () => {
+ const __logger = _logger.child({
+ module: "extract-worker",
+ method: "crawlFinishWorker",
+ });
+
+ let noJobTimeout = 1500;
+
+ while (!isShuttingDown) {
+ const job = await crawlFinishedQueue.getJobToProcess();
+
+ if (job === null) {
+ __logger.info("No jobs to process", { module: "nuq/metrics" });
+ await new Promise(resolve => setTimeout(resolve, noJobTimeout));
+ if (!process.env.NUQ_RABBITMQ_URL) {
+ noJobTimeout = Math.min(noJobTimeout * 2, 10000);
+ }
+ continue;
+ }
+
+ noJobTimeout = 500;
+
+ const logger = __logger.child({
+ zeroDataRetention: job.data?.zeroDataRetention ?? false,
+ crawlId: job.groupId,
+ });
+
+ logger.info("Acquired job");
+
+ const lockRenewInterval = setInterval(async () => {
+ logger.info("Renewing lock");
+ if (!(await crawlFinishedQueue.renewLock(job.id, job.lock!, logger))) {
+ logger.warn("Failed to renew lock");
+ clearInterval(lockRenewInterval);
+ return;
+ }
+ logger.info("Renewed lock");
+ }, 15000);
+
+ let processResult:
+ | {
+ ok: true;
+ data: Awaited<ReturnType<typeof processFinishCrawlJobInternal>>;
+ }
+ | { ok: false; error: any };
+
+ try {
+ processResult = {
+ ok: true,
+ data: await processFinishCrawlJobInternal(job, logger),
+ };
+ } catch (error) {
+ processResult = { ok: false, error };
+ }
+
+ clearInterval(lockRenewInterval);
+
+ if (processResult.ok) {
+ if (
+ !(await crawlFinishedQueue.jobFinish(
+ job.id,
+ job.lock!,
+ processResult.data,
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update job status");
+ }
+ } else {
+ if (
+ !(await crawlFinishedQueue.jobFail(
+ job.id,
+ job.lock!,
+ processResult.error instanceof Error
+ ? processResult.error.message
+ : typeof processResult.error === "string"
+ ? processResult.error
+ : JSON.stringify(processResult.error),
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update job status");
+ }
+ }
+ }
+};
+
// Start all workers
const app = Express();
@@ -297,7 +412,10 @@ app.listen(workerPort, () => {
process.exit(1);
});
- await Promise.all([workerFun(getExtractQueue(), processExtractJobInternal)]);
+ await Promise.all([
+ workerFun(getExtractQueue(), processExtractJobInternal),
+ crawlFinishWorker(),
+ ]);
_logger.info("All workers exited. Waiting for all jobs to finish...");
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index e662e999b..b78be69f6 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -37,6 +37,8 @@ import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";
import { _addScrapeJobToBullMQ } from "../queue-jobs";
import { BullMQOtel } from "bullmq-otel";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
+import { crawlGroup } from "../worker/nuq";
+import { getACUCTeam } from "../../controllers/auth";
const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
const workerStalledCheckInterval =
@@ -490,6 +492,15 @@ const processPrecrawlJob = async (token: string, job: Job) => {
// });
// }
+ await crawlGroup.addGroup(
+ crawlId,
+ sc.team_id,
+ ((await getACUCTeam(sc.team_id))?.flags?.crawlTtlHours ?? 24) *
+ 60 *
+ 60 *
+ 1000,
+ );
+
await saveCrawl(crawlId, sc);
await _addScrapeJobToBullMQ(
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index af3489d44..eda2398f7 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -1,320 +1,179 @@
import { logger as _logger } from "../../lib/logger";
import {
- addCrawlJobs,
finishCrawl,
getCrawlJobs,
getDoneJobsOrderedLength,
- lockURLsIndividually,
- StoredCrawl,
- unPreFinishCrawl,
} from "../../lib/crawl-redis";
-import { redisEvictConnection } from "../redis";
import { getCrawl } from "../../lib/crawl-redis";
-import { finishCrawlPre } from "../../lib/crawl-redis";
-import { getACUCTeam } from "../../controllers/auth";
-import { crawlToCrawler } from "../../lib/crawl-redis";
import { supabase_service } from "../supabase";
-import { v4 as uuidv4 } from "uuid";
-import { addScrapeJobs } from "../queue-jobs";
import { getJobs } from "../../controllers/v1/crawl-status";
import { logJob } from "../logging/log_job";
-import { createWebhookSender, WebhookEvent } from "../webhook";
-import { hasFormatOfType } from "../../lib/format-utils";
+import { createWebhookSender, WebhookEvent } from "../webhook/index";
import type { NuQJob } from "./nuq";
-import { ScrapeJobData } from "../../types";
-export async function finishCrawlIfNeeded(
- job: NuQJob<ScrapeJobData>,
- sc: StoredCrawl,
-) {
- const crawlId = job.data.crawl_id!;
+export async function finishCrawlSuper(job: NuQJob<any>) {
+ const crawlId = job.groupId;
if (!crawlId) {
return;
}
+ const sc = await getCrawl(crawlId);
+
+ if (!sc) {
+ return;
+ }
+
const logger = _logger.child({
module: "queue-worker",
- method: "finishCrawlIfNeeded",
+ method: "finishCrawl",
jobId: job.id,
scrapeId: job.id,
crawlId,
zeroDataRetention: sc.internalOptions.zeroDataRetention,
});
- if (await finishCrawlPre(crawlId, logger)) {
- logger.info("Crawl is pre-finished, checking if we need to add more jobs");
- if (
- sc.crawlerOptions &&
- !(await redisEvictConnection.exists(
- "crawl:" + crawlId + ":invisible_urls",
- ))
- ) {
- await redisEvictConnection.set(
- "crawl:" + crawlId + ":invisible_urls",
- "done",
- "EX",
- 60 * 60 * 24,
- );
-
- const sc = (await getCrawl(crawlId))!;
-
- const visitedUrls = new Set(
- await redisEvictConnection.smembers(
- "crawl:" + crawlId + ":visited_unique",
- ),
- );
-
- logger.info("Visited URLs", {
- visitedUrls: visitedUrls.size,
- });
-
- let lastUrls: string[] = [];
- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";
- if (
- useDbAuthentication &&
- hasFormatOfType(sc.scrapeOptions.formats, "changeTracking")
- ) {
- lastUrls = (
- (
- await supabase_service.rpc("diff_get_last_crawl_urls", {
- i_team_id: job.data.team_id,
- i_url: sc.originUrl!,
- })
- ).data ?? []
- ).map(x => x.url);
- }
-
- const lastUrlsSet = new Set(lastUrls);
-
- logger.info("Last URLs", {
- lastUrls: lastUrlsSet.size,
+ logger.info("Finishing crawl");
+ await finishCrawl(crawlId, logger);
+
+ if (!job.data.v1) {
+ const jobIDs = await getCrawlJobs(crawlId);
+
+ const jobs = (await getJobs(jobIDs)).sort(
+ (a, b) => a.timestamp - b.timestamp,
+ );
+ // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));
+ const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")
+ ? "failed"
+ : "completed";
+
+ const fullDocs = jobs
+ .map(x =>
+ x.returnvalue
+ ? Array.isArray(x.returnvalue)
+ ? x.returnvalue[0]
+ : x.returnvalue
+ : null,
+ )
+ .filter(x => x !== null);
+
+ await logJob(
+ {
+ job_id: crawlId,
+ success: jobStatus === "completed",
+ message: sc.cancelled ? "Cancelled" : undefined,
+ num_docs: fullDocs.length,
+ docs: [],
+ time_taken: (Date.now() - sc.createdAt) / 1000,
+ team_id: job.data.team_id,
+ mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",
+ url: sc.originUrl!,
+ scrapeOptions: sc.scrapeOptions,
+ crawlerOptions: sc.crawlerOptions,
+ origin: sc.originUrl!,
+ integration: job.data.integration,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ false,
+ sc.internalOptions?.bypassBilling ?? false,
+ );
+
+ // v0 web hooks, call when done with all the data
+ if (!job.data.v1) {
+ const sender = await createWebhookSender({
+ teamId: job.data.team_id,
+ jobId: crawlId,
+ webhook: job.data.webhook,
+ v0: true,
});
-
- const crawler = crawlToCrawler(
- crawlId,
- sc,
- (await getACUCTeam(job.data.team_id!))?.flags ?? null,
- sc.originUrl!,
- sc.crawlerOptions,
- );
-
- const univistedUrls = await crawler.filterLinks(
- Array.from(lastUrlsSet).filter(x => !visitedUrls.has(x)),
- Infinity,
- sc.crawlerOptions.maxDepth ?? 10,
- );
-
- const addableJobCount =
- sc.crawlerOptions.limit === undefined
- ? Infinity
- : sc.crawlerOptions.limit - (await getDoneJobsOrderedLength(crawlId));
-
- if (univistedUrls.links.length !== 0 && addableJobCount > 0) {
- logger.info("Adding jobs", {
- univistedUrls: univistedUrls.links.length,
- addableJobCount,
- });
-
- const jobs = univistedUrls.links.slice(0, addableJobCount).map(url => {
- const uuid = uuidv4();
- return {
- jobId: uuid,
- data: {
- url,
- mode: "single_urls" as const,
- team_id: job.data.team_id,
- crawlerOptions: {
- ...sc.crawlerOptions,
- urlInvisibleInCurrentCrawl: true,
- },
- scrapeOptions: sc.scrapeOptions,
- internalOptions: sc.internalOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: crawlId,
- sitemapped: true,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- apiKeyId: job.data.apiKeyId,
- },
- priority: 20, // TODO: make this dynamic
- };
- });
-
- const lockedIds = await lockURLsIndividually(
- crawlId,
- sc,
- jobs.map(x => ({ id: x.jobId, url: x.data.url })),
- );
- const lockedJobs = jobs.filter(x =>
- lockedIds.find(y => y.id === x.jobId),
- );
- await addCrawlJobs(
- crawlId,
- lockedJobs.map(x => x.jobId),
- logger,
- );
- await addScrapeJobs(lockedJobs);
-
- if (lockedJobs.length > 0) {
- logger.info("Added jobs, not going for the full finish", {
- lockedJobs: lockedJobs.length,
+ if (sender) {
+ const documents = fullDocs.map((doc: any) => ({
+ content: {
+ content: doc?.content ?? doc?.rawHtml ?? doc?.markdown ?? "",
+ markdown: doc?.markdown,
+ metadata: doc?.metadata ?? {},
+ },
+ source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
+ }));
+ if (sc.crawlerOptions !== null) {
+ sender.send(WebhookEvent.CRAWL_COMPLETED, {
+ success: true,
+ data: documents,
});
-
- await unPreFinishCrawl(crawlId);
- return;
} else {
- logger.info(
- "No jobs added (all discovered URLs were locked), finishing crawl",
- );
+ sender.send(WebhookEvent.BATCH_SCRAPE_COMPLETED, {
+ success: true,
+ data: documents,
+ });
}
}
}
+ } else {
+ const num_docs = await getDoneJobsOrderedLength(crawlId);
+ const jobStatus = sc.cancelled ? "failed" : "completed";
- logger.info("Finishing crawl");
- await finishCrawl(crawlId, logger);
-
- if (!job.data.v1) {
- const jobIDs = await getCrawlJobs(crawlId);
-
- const jobs = (await getJobs(jobIDs)).sort(
- (a, b) => a.timestamp - b.timestamp,
- );
- // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));
- const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")
- ? "failed"
- : "completed";
+ let credits_billed = null;
- const fullDocs = jobs
- .map(x =>
- x.returnvalue
- ? Array.isArray(x.returnvalue)
- ? x.returnvalue[0]
- : x.returnvalue
- : null,
- )
- .filter(x => x !== null);
-
- await logJob(
+ if (process.env.USE_DB_AUTHENTICATION === "true") {
+ const creditsRpc = await supabase_service.rpc(
+ "credits_billed_by_crawl_id_1",
{
- job_id: crawlId,
- success: jobStatus === "completed",
- message: sc.cancelled ? "Cancelled" : undefined,
- num_docs: fullDocs.length,
- docs: [],
- time_taken: (Date.now() - sc.createdAt) / 1000,
- team_id: job.data.team_id,
- mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",
- url: sc.originUrl!,
- scrapeOptions: sc.scrapeOptions,
- crawlerOptions: sc.crawlerOptions,
- origin: sc.originUrl!,
- integration: job.data.integration,
- zeroDataRetention: job.data.zeroDataRetention,
+ i_crawl_id: crawlId,
},
- false,
- sc.internalOptions?.bypassBilling ?? false,
);
- // v0 web hooks, call when done with all the data
- if (!job.data.v1) {
- const sender = await createWebhookSender({
- teamId: job.data.team_id,
- jobId: crawlId,
- webhook: job.data.webhook,
- v0: true,
+ credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;
+
+ if (credits_billed === null) {
+ logger.warn("Credits billed is null", {
+ error: creditsRpc.error,
});
- if (sender) {
- const documents = fullDocs.map((doc: any) => ({
- content: {
- content: doc?.content ?? doc?.rawHtml ?? doc?.markdown ?? "",
- markdown: doc?.markdown,
- metadata: doc?.metadata ?? {},
- },
- source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
- }));
- if (sc.crawlerOptions !== null) {
- sender.send(WebhookEvent.CRAWL_COMPLETED, {
- success: true,
- data: documents,
- });
- } else {
- sender.send(WebhookEvent.BATCH_SCRAPE_COMPLETED, {
- success: true,
- data: documents,
- });
- }
- }
}
- } else {
- const num_docs = await getDoneJobsOrderedLength(crawlId);
- const jobStatus = sc.cancelled ? "failed" : "completed";
-
- let credits_billed = null;
-
- if (process.env.USE_DB_AUTHENTICATION === "true") {
- const creditsRpc = await supabase_service.rpc(
- "credits_billed_by_crawl_id_1",
- {
- i_crawl_id: crawlId,
- },
- );
-
- credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;
+ }
- if (credits_billed === null) {
- logger.warn("Credits billed is null", {
- error: creditsRpc.error,
+ await logJob(
+ {
+ job_id: crawlId,
+ success: jobStatus === "completed",
+ message: sc.cancelled ? "Cancelled" : undefined,
+ num_docs,
+ docs: [],
+ time_taken: (Date.now() - sc.createdAt) / 1000,
+ team_id: sc.team_id,
+ scrapeOptions: sc.scrapeOptions,
+ mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",
+ url:
+ sc?.originUrl ??
+ (sc.crawlerOptions === null ? "Batch Scrape" : "Unknown"),
+ crawlerOptions: sc.crawlerOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ credits_billed,
+ zeroDataRetention: job.data.zeroDataRetention,
+ },
+ true,
+ sc.internalOptions?.bypassBilling ?? false,
+ );
+
+ // v1 web hooks, call when done with no data, but with event completed
+ if (job.data.v1 && job.data.webhook) {
+ const sender = await createWebhookSender({
+ teamId: job.data.team_id,
+ jobId: crawlId,
+ webhook: job.data.webhook,
+ v0: false,
+ });
+ if (sender) {
+ if (sc.crawlerOptions !== null) {
+ sender.send(WebhookEvent.CRAWL_COMPLETED, {
+ success: true,
+ data: [],
+ });
+ } else {
+ sender.send(WebhookEvent.BATCH_SCRAPE_COMPLETED, {
+ success: true,
+ data: [],
});
- }
- }
-
- await logJob(
- {
- job_id: crawlId,
- success: jobStatus === "completed",
- message: sc.cancelled ? "Cancelled" : undefined,
- num_docs,
- docs: [],
- time_taken: (Date.now() - sc.createdAt) / 1000,
- team_id: sc.team_id,
- scrapeOptions: sc.scrapeOptions,
- mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",
- url:
- sc?.originUrl ??
- (sc.crawlerOptions === null ? "Batch Scrape" : "Unknown"),
- crawlerOptions: sc.crawlerOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- credits_billed,
- zeroDataRetention: job.data.zeroDataRetention,
- },
- true,
- sc.internalOptions?.bypassBilling ?? false,
- );
-
- // v1 web hooks, call when done with no data, but with event completed
- if (job.data.v1 && job.data.webhook) {
- const sender = await createWebhookSender({
- teamId: job.data.team_id,
- jobId: crawlId,
- webhook: job.data.webhook,
- v0: false,
- });
- if (sender) {
- if (sc.crawlerOptions !== null) {
- sender.send(WebhookEvent.CRAWL_COMPLETED, {
- success: true,
- data: [],
- });
- } else {
- sender.send(WebhookEvent.BATCH_SCRAPE_COMPLETED, {
- success: true,
- data: [],
- });
- }
}
}
}
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index 8c5126cd9..a0463dd2d 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -499,6 +499,95 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
}
+ public async getGroupAnyJob(
+ groupId: string,
+ ownerId: string,
+ ): Promise<NuQJob<JobData, JobReturnValue> | null> {
+ return this.rowToJob(
+ (
+ await nuqPool.query(
+ `
+ SELECT ${this.jobReturning.join(", ")}
+ FROM ${this.queueName}
+ WHERE ${this.queueName}.group_id = $1
+ AND ${this.queueName}.owner_id = $2
+ AND ${this.queueName}.data->>'mode' = 'single_urls'
+ LIMIT 1;
+ `,
+ [groupId, normalizeOwnerId(ownerId)],
+ )
+ ).rows[0],
+ );
+ }
+
+ public async getGroupNumericStats(
+ groupId: string,
+ _logger: Logger = logger,
+ ): Promise<Record<NuQJobStatus, number>> {
+ const start = Date.now();
+ try {
+ return Object.fromEntries(
+ (
+ await nuqPool.query(
+ `
+ SELECT ${this.queueName}.status::text as status, COUNT(*) as count
+ FROM ${this.queueName}
+ WHERE ${this.queueName}.group_id = $1
+ AND ${this.queueName}.data->>'mode' = 'single_urls'
+ GROUP BY ${this.queueName}.status
+ UNION ALL
+ SELECT 'backlog'::text as status, COUNT(*) as count
+ FROM ${this.queueName}_backlog
+ WHERE ${this.queueName}_backlog.group_id = $1
+ AND ${this.queueName}_backlog.data->>'mode' = 'single_urls'
+ `,
+ [groupId],
+ )
+ ).rows.map(row => [row.status, parseInt(row.count, 10)]),
+ );
+ } finally {
+ _logger.info("nuqGetGroupNumericStats metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetGroupNumericStats",
+ duration: Date.now() - start,
+ crawlId: groupId,
+ });
+ }
+ }
+
+ // TODO: make more generalizable
+ public async getCrawlJobsForListing(
+ groupId: string,
+ limit: number,
+ offset: number,
+ _logger: Logger = logger,
+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {
+ const start = Date.now();
+ try {
+ return (
+ await nuqPool.query(
+ `
+ SELECT ${this.jobReturning.join(", ")}
+ FROM ${this.queueName}
+ WHERE ${this.queueName}.group_id = $1
+ AND ${this.queueName}.status = 'completed'
+ AND ${this.queueName}.data->>'mode' = 'single_urls'
+ ORDER BY finished_at ASC, created_at ASC
+ LIMIT $2 OFFSET $3;
+ `,
+ [groupId, limit, offset],
+ )
+ ).rows.map(row => this.rowToJob(row)!);
+ } finally {
+ _logger.info("nuqGetCrawlJobsForListing metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetCrawlJobsForListing",
+ duration: Date.now() - start,
+ crawlId: groupId,
+ });
+ }
+ }
+
public async removeJob(
id: string,
_logger: Logger = logger,
@@ -1190,6 +1279,169 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
}
+// === Group Management
+
+export type NuQGroupStatus = "active" | "completed" | "cancelled";
+
+export type NuQJobGroupInstance = {
+ id: string;
+ status: NuQGroupStatus;
+ createdAt: Date;
+ ownerId: string;
+ ttl: number;
+ expiresAt?: Date;
+};
+
+class NuQJobGroup {
+ constructor(public readonly groupName: string) {}
+
+ private readonly groupReturning = [
+ "id",
+ "status",
+ "created_at",
+ "owner_id",
+ "ttl",
+ "expires_at",
+ ];
+
+ private rowToGroup(row: any): NuQJobGroupInstance | null {
+ if (!row) return null;
+ return {
+ id: row.id,
+ status: row.status,
+ createdAt: new Date(row.created_at),
+ ownerId: row.owner_id,
+ ttl: row.ttl,
+ expiresAt: row.expires_at ? new Date(row.expires_at) : undefined,
+ };
+ }
+
+ public async addGroup(
+ id: string,
+ ownerId: string,
+ ttl?: number,
+ _logger: Logger = logger,
+ ): Promise<NuQJobGroupInstance> {
+ return withSpan("nuq.addGroup", async span => {
+ setSpanAttributes(span, {
+ "nuq.group_name": this.groupName,
+ "nuq.group_id": id,
+ "nuq.ttl": ttl ?? 86400000,
+ });
+
+ const start = Date.now();
+ try {
+ const result = this.rowToGroup(
+ (
+ await nuqPool.query(
+ `INSERT INTO ${this.groupName} (id, owner_id, ttl) VALUES ($1, $2, $3) RETURNING ${this.groupReturning.join(", ")};`,
+ [id, normalizeOwnerId(ownerId), ttl ?? 86400000],
+ )
+ ).rows[0],
+ )!;
+
+ setSpanAttributes(span, {
+ "nuq.group_created": true,
+ });
+
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ _logger.info("nuqAddGroup metrics", {
+ module: "nuq/metrics",
+ method: "nuqAddGroup",
+ duration,
+ groupId: id,
+ });
+ }
+ });
+ }
+
+ public async getGroup(
+ id: string,
+ _logger: Logger = logger,
+ ): Promise<NuQJobGroupInstance | null> {
+ return withSpan("nuq.getGroup", async span => {
+ setSpanAttributes(span, {
+ "nuq.group_name": this.groupName,
+ "nuq.group_id": id,
+ });
+
+ const start = Date.now();
+ try {
+ const result = this.rowToGroup(
+ (
+ await nuqPool.query(
+ `SELECT ${this.groupReturning.join(", ")} FROM ${this.groupName} WHERE ${this.groupName}.id = $1;`,
+ [id],
+ )
+ ).rows[0],
+ );
+
+ setSpanAttributes(span, {
+ "nuq.group_found": result !== null,
+ "nuq.group_status": result?.status,
+ });
+
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ _logger.info("nuqGetGroup metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetGroup",
+ duration,
+ groupId: id,
+ });
+ }
+ });
+ }
+
+ public async getOngoingByOwner(
+ ownerId: string,
+ _logger: Logger = logger,
+ ): Promise<NuQJobGroupInstance[]> {
+ return withSpan("nuq.getGroupByOwner", async span => {
+ setSpanAttributes(span, {
+ "nuq.group_name": this.groupName,
+ "nuq.owner_id": ownerId,
+ });
+
+ const start = Date.now();
+ try {
+ const result = (
+ await nuqPool.query(
+ `SELECT ${this.groupReturning.join(", ")} FROM ${this.groupName} WHERE ${this.groupName}.owner_id = $1 AND ${this.groupName}.status = 'active'`,
+ [normalizeOwnerId(ownerId)],
+ )
+ ).rows.map(x => this.rowToGroup(x)!);
+
+ setSpanAttributes(span, {
+ "nuq.groups_found": result.length,
+ });
+
+ return result;
+ } finally {
+ const duration = Date.now() - start;
+ setSpanAttributes(span, {
+ "nuq.duration_ms": duration,
+ });
+ _logger.info("nuqGetGroup metrics", {
+ module: "nuq/metrics",
+ method: "nuqGetGroupByOwner",
+ duration,
+ ownerId: ownerId,
+ });
+ }
+ });
+ }
+}
+
export function nuqGetLocalMetrics(): string {
return `# HELP nuq_pool_waiting_count Number of requests waiting in the pool\n# TYPE nuq_pool_waiting_count gauge\nnuq_pool_waiting_count ${nuqPool.waitingCount}\n
# HELP nuq_pool_idle_count Number of connections idle in the pool\n# TYPE nuq_pool_idle_count gauge\nnuq_pool_idle_count ${nuqPool.idleCount}\n
@@ -1214,6 +1466,9 @@ export async function nuqHealthCheck(): Promise<boolean> {
export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape", {
backlog: true,
});
+export const crawlFinishedQueue = new NuQ("nuq.queue_crawl_finished", {});
+
+export const crawlGroup = new NuQJobGroup("nuq.group_crawl");
// === Cleanup
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index 0d4804b82..db1b40f56 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -50,7 +50,6 @@ import { WebCrawler } from "../../scraper/WebScraper/crawler";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { getBillingQueue } from "../queue-service";
import type { Logger } from "winston";
-import { finishCrawlIfNeeded } from "./crawl-logic";
import {
RacedRedirectError,
ScrapeJobTimeoutError,
@@ -491,8 +490,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
logger.debug("Declaring job as done...");
await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);
-
- await finishCrawlIfNeeded(job, sc);
} else {
try {
signal?.throwIfAborted();
@@ -569,8 +566,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
"crawl:" + job.data.crawl_id + ":jobs_qualified",
job.id,
);
-
- await finishCrawlIfNeeded(job, sc);
}
const isEarlyTimeout = error instanceof ScrapeJobTimeoutError;
@@ -937,16 +932,12 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
logger.debug("Done queueing jobs!");
await finishCrawlKickoff(job.data.crawl_id);
- await finishCrawlIfNeeded(job, sc);
return { success: true };
} catch (error) {
logger.error("An error occurred!", { error });
await finishCrawlKickoff(job.data.crawl_id);
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
- if (sc) {
- await finishCrawlIfNeeded(job, sc);
- }
return { success: false, error };
}
}
@@ -1066,10 +1057,6 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
"crawl:" + job.data.crawl_id + ":sitemap_jobs_done",
24 * 60 * 60,
);
-
- if (sc) {
- await finishCrawlIfNeeded(job, sc);
- }
}
}
diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sql
index 0252dec84..119ec8d27 100644
--- a/apps/nuq-postgres/nuq.sql
+++ b/apps/nuq-postgres/nuq.sql
@@ -9,6 +9,12 @@ EXCEPTION
WHEN duplicate_object THEN null;
END $$;
+DO $$ BEGIN
+ CREATE TYPE nuq.group_status AS ENUM ('active', 'completed', 'cancelled');
+EXCEPTION
+ WHEN duplicate_object THEN null;
+END $$;
+
CREATE TABLE IF NOT EXISTS nuq.queue_scrape (
id uuid NOT NULL DEFAULT gen_random_uuid(),
status nuq.job_status NOT NULL DEFAULT 'queued'::nuq.job_status,
@@ -51,11 +57,11 @@ CREATE TABLE IF NOT EXISTS nuq.queue_scrape_backlog (
);
SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$
- DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour';
+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour' AND group_id IS NULL;
$$);
SELECT cron.schedule('nuq_queue_scrape_clean_failed', '*/5 * * * *', $$
- DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '6 hours';
+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '6 hours' AND group_id IS NULL;
$$);
SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$
@@ -64,6 +70,107 @@ SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$
SELECT pg_notify('nuq.queue_scrape', (id::text || '|' || 'failed'::text)) FROM stallfail;
$$);
+SELECT cron.schedule('nuq_queue_scrape_backlog_reaper', '* * * * *', $$
+ DELETE FROM nuq.queue_scrape_backlog
+ WHERE nuq.queue_scrape_backlog.times_out_at < now();
+$$);
+
SELECT cron.schedule('nuq_queue_scrape_reindex', '0 9 * * *', $$
REINDEX TABLE CONCURRENTLY nuq.queue_scrape;
$$);
+
+CREATE TABLE IF NOT EXISTS nuq.queue_crawl_finished (
+ id uuid NOT NULL DEFAULT gen_random_uuid(),
+ status nuq.job_status NOT NULL DEFAULT 'queued'::nuq.job_status,
+ data jsonb,
+ created_at timestamp with time zone NOT NULL DEFAULT now(),
+ priority int NOT NULL DEFAULT 0,
+ lock uuid,
+ locked_at timestamp with time zone,
+ stalls integer,
+ finished_at timestamp with time zone,
+ listen_channel_id text, -- for listenable jobs over rabbitmq
+ returnvalue jsonb, -- only for selfhost
+ failedreason text, -- only for selfhost
+ owner_id uuid,
+ group_id uuid,
+ CONSTRAINT queue_crawl_finished_pkey PRIMARY KEY (id)
+);
+
+ALTER TABLE nuq.queue_crawl_finished
+SET (autovacuum_vacuum_scale_factor = 0.01,
+ autovacuum_analyze_scale_factor = 0.01,
+ autovacuum_vacuum_cost_limit = 2000,
+ autovacuum_vacuum_cost_delay = 2);
+
+CREATE INDEX IF NOT EXISTS queue_crawl_finished_active_locked_at_idx ON nuq.queue_crawl_finished USING btree (locked_at) WHERE (status = 'active'::nuq.job_status);
+CREATE INDEX IF NOT EXISTS nuq_queue_crawl_finished_queued_optimal_2_idx ON nuq.queue_crawl_finished (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status);
+CREATE INDEX IF NOT EXISTS nuq_queue_crawl_finished_failed_created_at_idx ON nuq.queue_crawl_finished USING btree (created_at) WHERE (status = 'failed'::nuq.job_status);
+CREATE INDEX IF NOT EXISTS nuq_queue_crawl_finished_completed_created_at_idx ON nuq.queue_crawl_finished USING btree (created_at) WHERE (status = 'completed'::nuq.job_status);
+
+SELECT cron.schedule('nuq_queue_crawl_finished_clean_completed', '*/5 * * * *', $$
+ DELETE FROM nuq.queue_crawl_finished WHERE nuq.queue_crawl_finished.status = 'completed'::nuq.job_status AND nuq.queue_crawl_finished.created_at < now() - interval '1 hour' AND group_id IS NULL;
+$$);
+
+SELECT cron.schedule('nuq_queue_crawl_finished_clean_failed', '*/5 * * * *', $$
+ DELETE FROM nuq.queue_crawl_finished WHERE nuq.queue_crawl_finished.status = 'failed'::nuq.job_status AND nuq.queue_crawl_finished.created_at < now() - interval '6 hours' AND group_id IS NULL;
+$$);
+
+SELECT cron.schedule('nuq_queue_crawl_finished_lock_reaper', '15 seconds', $$
+ UPDATE nuq.queue_crawl_finished SET status = 'queued'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_crawl_finished.locked_at <= now() - interval '1 minute' AND nuq.queue_crawl_finished.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_crawl_finished.stalls, 0) < 9;
+ WITH stallfail AS (UPDATE nuq.queue_crawl_finished SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_crawl_finished.locked_at <= now() - interval '1 minute' AND nuq.queue_crawl_finished.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_crawl_finished.stalls, 0) >= 9 RETURNING id)
+ SELECT pg_notify('nuq.queue_crawl_finished', (id::text || '|' || 'failed'::text)) FROM stallfail;
+$$);
+
+SELECT cron.schedule('nuq_queue_crawl_finished_reindex', '0 9 * * *', $$
+ REINDEX TABLE CONCURRENTLY nuq.queue_crawl_finished;
+$$);
+
+CREATE TABLE IF NOT EXISTS nuq.group_crawl (
+ id uuid NOT NULL,
+ status nuq.group_status NOT NULL DEFAULT 'active'::nuq.group_status,
+ created_at timestamptz NOT NULL DEFAULT now(),
+ owner_id uuid NOT NULL,
+ ttl int8 NOT NULL DEFAULT 86400000,
+ expires_at timestamptz,
+ CONSTRAINT group_crawl_pkey PRIMARY KEY (id)
+);
+
+SELECT cron.schedule('nuq_group_crawl_finished', '15 seconds', $$
+ WITH finished_groups AS (
+ UPDATE nuq.group_crawl
+ SET status = 'completed'::nuq.group_status, expires_at = now() + MAKE_INTERVAL(secs => nuq.group_crawl.ttl / 1000)
+ WHERE nuq.group_crawl.status = 'active'::nuq.group_status
+ AND NOT EXISTS (
+ SELECT 1 FROM nuq.queue_scrape
+ WHERE nuq.queue_scrape.status IN ('active', 'queued')
+ AND nuq.queue_scrape.group_id = nuq.group_crawl.id
+ )
+ AND NOT EXISTS (
+ SELECT 1 FROM nuq.queue_scrape_backlog
+ WHERE nuq.queue_scrape_backlog.group_id = nuq.group_crawl.id
+ )
+ RETURNING id, owner_id
+ )
+ INSERT INTO nuq.queue_crawl_finished (data, owner_id, group_id)
+ SELECT '{}'::jsonb, finished_groups.owner_id, finished_groups.id
+ FROM finished_groups;
+$$);
+
+SELECT cron.schedule('nuq_group_crawl_clean', '*/5 * * * *', $$
+ WITH cleaned_groups AS (
+ DELETE FROM nuq.group_crawl
+ WHERE nuq.group_crawl.status = 'completed'::nuq.group_status
+ AND nuq.group_crawl.expires_at < now();
+ RETURNING *;
+ ), cleaned_jobs_queue_scrape AS (
+ DELETE FROM nuq.queue_scrape
+ WHERE nuq.queue_scrape.group_id IN (SELECT id FROM cleaned_groups)
+ ), cleaned_jobs_queue_scrape_backlog AS (
+ DELETE FROM nuq.queue_scrape_backlog
+ WHERE nuq.queue_scrape_backlog.group_id IN (SELECT id FROM cleaned_groups)
+ ), cleaned_jobs_crawl_finished AS (
+ DELETE FROM nuq.queue_crawl_finished
+ WHERE nuq.queue_crawl_finished.group_id IN (SELECT id FROM cleaned_groups)
+ );
+$$);