Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: new crawl architecture (#2320)
Gergő Móricz54f148204ff7
diff --git a/apps/api/src/__tests__/snips/v1/crawl.test.ts b/apps/api/src/__tests__/snips/v1/crawl.test.tsindex 8bb55d43b..4be811e02 100644--- a/apps/api/src/__tests__/snips/v1/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v1/crawl.test.ts@@ -166,6 +166,9 @@ describeIf(ALLOW_TEST_SUITE_WEBSITE)("Crawl tests", () => {await asyncCrawlWaitForFinish(res.id, identity);+ // wait for crawl finish to happen on DB cron+ await new Promise(resolve => setTimeout(resolve, 15000));+const ongoing2 = await crawlOngoing(identity);expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined();diff --git a/apps/api/src/__tests__/snips/v2/crawl.test.ts b/apps/api/src/__tests__/snips/v2/crawl.test.tsindex 54cf66cdb..19e4bcbf6 100644--- a/apps/api/src/__tests__/snips/v2/crawl.test.ts+++ b/apps/api/src/__tests__/snips/v2/crawl.test.ts@@ -172,11 +172,14 @@ describe("Crawl tests", () => {await asyncCrawlWaitForFinish(res.id, identity);+ // wait for crawl finish to happen on DB cron+ await new Promise(resolve => setTimeout(resolve, 15000));+const ongoing2 = await crawlOngoing(identity);expect(ongoing2.crawls.find(x => x.id === res.id)).toBeUndefined();},- 3 * scrapeTimeout,+ 3 * scrapeTimeout + 15000,);// TEMP: Flakydiff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex 59c38eb89..8c795f06f 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -33,6 +33,7 @@ import { ZodError } from "zod";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { fromV0ScrapeOptions } from "../v2/types";import { isSelfHosted } from "../../lib/deployment";+import { crawlGroup } from "../../services/worker/nuq";export async function crawlController(req: Request, res: Response) {try {@@ -203,6 +204,12 @@ export async function crawlController(req: Request, res: Response) {sc.robots = await crawler.getRobotsTxt();} catch (_) {}+ await crawlGroup.addGroup(+ id,+ sc.team_id,+ (chunk?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ );+await saveCrawl(id, sc);await markCrawlActive(id);diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex c12ff7a9f..233e367d1 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -25,6 +25,7 @@ import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";+import { crawlGroup } from "../../services/worker/nuq";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -136,6 +137,11 @@ export async function batchScrapeController(};if (!req.body.appendToId) {+ await crawlGroup.addGroup(+ id,+ sc.team_id,+ (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ );await saveCrawl(id, sc);await markCrawlActive(id);}diff --git a/apps/api/src/controllers/v1/crawl-ongoing.ts b/apps/api/src/controllers/v1/crawl-ongoing.tsindex 592d31dce..74f7143be 100644--- a/apps/api/src/controllers/v1/crawl-ongoing.ts+++ b/apps/api/src/controllers/v1/crawl-ongoing.ts@@ -4,15 +4,18 @@ import {RequestWithAuth,toNewCrawlerOptions,} from "./types";-import { getCrawl, getCrawlsByTeamId } from "../../lib/crawl-redis";+import { getCrawl } from "../../lib/crawl-redis";import { configDotenv } from "dotenv";+import { crawlGroup } from "../../services/worker/nuq";configDotenv();export async function ongoingCrawlsController(req: RequestWithAuth<{}, undefined, OngoingCrawlsResponse>,res: Response<OngoingCrawlsResponse>,) {- const ids = await getCrawlsByTeamId(req.auth.team_id);+ const ids = (await crawlGroup.getOngoingByOwner(req.auth.team_id)).map(+ x => x.id,+ );const crawls = (await Promise.all(ids.map(async id => ({ ...(await getCrawl(id)), id })))diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 673d9d315..15ed8ac7c 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -21,7 +21,12 @@ import { configDotenv } from "dotenv";import { logger } from "../../lib/logger";import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getJobFromGCS } from "../../lib/gcs-jobs";-import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";+import {+ scrapeQueue,+ NuQJob,+ NuQJobStatus,+ crawlGroup,+} from "../../services/worker/nuq";import { ScrapeJobSingleUrls } from "../../types";configDotenv();@@ -165,291 +170,109 @@ export async function crawlStatusController(? start + parseInt(req.query.limit, 10) - 1: undefined;+ const group = await crawlGroup.getGroup(req.params.jobId);+ const groupAnyJob = await scrapeQueue.getGroupAnyJob(+ req.params.jobId,+ req.auth.team_id,+ );const sc = await getCrawl(req.params.jobId);- const djoCutoff = Date.now() - 250;+ if (!group || (!groupAnyJob && (!sc || sc.team_id !== req.auth.team_id))) {+ return res.status(404).json({ success: false, error: "Job not found" });+ }- if (sc) {- if (sc.team_id !== req.auth.team_id) {- return res.status(404).json({ success: false, error: "Job not found" });- }- } else if (process.env.USE_DB_AUTHENTICATION === "true") {- const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service- .from("firecrawl_jobs")- .select("*")- .eq("job_id", req.params.jobId)- .limit(1);-- let crawlAdded: Date;- let crawlTeamId: string;-- if (crawlJobError || !crawlJobs || crawlJobs.length === 0) {- const { data: scrapeJobs, error: scrapeJobError } =- await supabase_rr_service- .from("firecrawl_jobs")- .select("*")- .eq("crawl_id", req.params.jobId)- .order("date_added", { ascending: false })- .limit(1);-- if (scrapeJobError || !scrapeJobs || scrapeJobs.length === 0) {- return res.status(404).json({ success: false, error: "Job not found" });- }-- const scrapeJob = scrapeJobs[0];- crawlAdded = new Date(scrapeJob.date_added); // use last scrape as date added for crawl- crawlTeamId = scrapeJob.team_id;- } else {- const crawlJob = crawlJobs[0];- crawlAdded = new Date(crawlJob.date_added);- crawlTeamId = crawlJob.team_id;- }+ const zeroDataRetention = !!(+ groupAnyJob?.data?.zeroDataRetention ?? sc?.zeroDataRetention+ );- if (crawlTeamId !== req.auth.team_id) {- return res.status(404).json({ success: false, error: "Job not found" });- }+ const numericStats = await scrapeQueue.getGroupNumericStats(+ req.params.jobId,+ logger.child({ zeroDataRetention }),+ );- const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;- const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;-- if (new Date().valueOf() - crawlAdded.valueOf() > crawlTtlMs) {- return res.status(404).json({ success: false, error: "Job expired" });- }- } else {- // if SC is gone and no DB, that means the job is expired, or never existed to begin with- return res.status(404).json({ success: false, error: "Job not found" });- }+ const creditsRpc =+ process.env.USE_DB_AUTHENTICATION === "true"+ ? await supabase_service.rpc(+ "credits_billed_by_crawl_id_1",+ {+ i_crawl_id: req.params.jobId,+ },+ { get: true },+ )+ : null;let outputBulkA: {status?: "completed" | "scraping" | "cancelled";completed?: number;total?: number;creditsUsed?: number;+ } = {+ status: group.status === "active" ? "scraping" : group.status,+ completed: numericStats.completed ?? 0,+ total:+ (numericStats.completed ?? 0) ++ (numericStats.active ?? 0) ++ (numericStats.queued ?? 0) ++ (numericStats.backlog ?? 0),+ creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,};- if (sc) {- // Local mode (easier, low-pressure)- const kickoffFinished = await isCrawlKickoffFinished(req.params.jobId);- let total = await getCrawlQualifiedJobCount(req.params.jobId);-- let completed = await getDoneJobsOrderedLength(req.params.jobId, djoCutoff);- let creditsUsed =- completed *- (sc.scrapeOptions?.formats?.find(- x => typeof x === "object" && x.type === "json",- )- ? 5- : 1);-- if (process.env.USE_DB_AUTHENTICATION === "true") {- const creditsRpc = await supabase_service.rpc(- "credits_billed_by_crawl_id_1",- {- i_crawl_id: req.params.jobId,- },- { get: true },- );-- creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? creditsUsed;-- if (- total === 0 &&- completed === 0 &&- Date.now() - sc.createdAt > 1000 * 60- ) {- const x = await supabase_service.rpc(- "crawl_status_job_count_1",- {- i_team_id: req.auth.team_id,- i_crawl_id: req.params.jobId,- },- { get: true },- );-- total = x.data?.[0]?.count ?? 0;- completed = total;- }- }-- outputBulkA = {- status: sc.cancelled- ? "cancelled"- : completed === total && kickoffFinished- ? "completed"- : "scraping",- total,- completed,- creditsUsed,- };- } else {- // DB must be specified at this point, otherwise control flow kills execution earlier- // DB mode (once job expires from Redis)- const { data: crawlJobCounts, error: crawlJobCountsError } =- await supabase_service.rpc(- "crawl_status_job_count_1",- {- i_team_id: req.auth.team_id,- i_crawl_id: req.params.jobId,- },- { get: true },- );-- if (crawlJobCountsError || !crawlJobCounts || crawlJobCounts.length === 0) {- logger.error("Error getting crawl job count", {- error: crawlJobCountsError,- });- throw new Error("Error getting crawl job count", {- cause: crawlJobCountsError,- });- }-- const crawlJobCount: number = crawlJobCounts[0].count ?? 0;-- const creditsRpc = await supabase_service.rpc(- "credits_billed_by_crawl_id_1",- {- i_crawl_id: req.params.jobId,- },- { get: true },- );-- const creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? crawlJobCount;-- outputBulkA = {- status: "completed", // Salvage expired job- total: crawlJobCount,- completed: crawlJobCount,- creditsUsed,- };- }-let outputBulkB: {data: Document[];next: string | undefined;};- if (sc || process.env.USE_DB_AUTHENTICATION !== "true") {- // old BullMQ-based path- const doneJobs = await getDoneJobsOrderedUntil(- req.params.jobId,- djoCutoff,- start,- end !== undefined ? end - start : 100,- );-- let scrapes: Document[] = [];- let iteratedOver = 0;- let bytes = 0;- const bytesLimit = 10485760; // 10 MiB in bytes-- for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) {- const jobIds = doneJobs.slice(i * 50, (i + 1) * 50);- const jobs = await getJobs(jobIds);-- for (const job of jobs) {- if (job.status === "failed") {- continue;- } else {- if (job?.returnvalue) {- scrapes.push(job.returnvalue);- bytes += JSON.stringify(job.returnvalue).length;- } else {- logger.warn(- "Job was considered done, but returnvalue is undefined!",- {- scrapeId: job.id,- crawlId: req.params.jobId,- state: job.status,- returnvalue: job?.returnvalue,- },- );- }-- iteratedOver++;- }-- if (bytes > bytesLimit) {- break;- }- }-- if (bytes > bytesLimit) {- break;- }- }-- if (bytes > bytesLimit && scrapes.length !== 1) {- scrapes.splice(scrapes.length - 1, 1);- iteratedOver--;- }-- outputBulkB = {- data: scrapes,- next:- (outputBulkA.total ?? 0) > start + iteratedOver- ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`- : undefined,- };- } else {- // new DB-based path- const { data, error } = await supabase_service.rpc(- "crawl_status_1",- {- i_team_id: req.auth.team_id,- i_crawl_id: req.params.jobId,- i_start: start,- i_end: end ?? start + 100,- },- { get: true },- );-- if (error || !data) {- logger.error("Error getting crawl status from DB", { error });- throw new Error("Error getting crawl status from DB", { cause: error });+ const doneJobs = await scrapeQueue.getCrawlJobsForListing(+ req.params.jobId,+ end !== undefined ? end - start : 100,+ start,+ logger.child({ zeroDataRetention }),+ );++ let scrapes: Document[] = [];+ let iteratedOver = 0;+ let bytes = 0;+ const bytesLimit = 10485760; // 10 MiB in bytes++ const scrapeBlobs = await Promise.all(+ doneJobs.map(+ async x =>+ [x.id, x.returnvalue ?? (await getJobFromGCS(x.id))?.[0]] as const,+ ),+ );++ for (const [id, scrape] of scrapeBlobs) {+ if (scrape) {+ scrapes.push(scrape);+ bytes += JSON.stringify(scrape).length;+ } else {+ logger.warn("Job was considered done, but returnvalue is undefined!", {+ jobId: id,+ returnvalue: scrape,+ zeroDataRetention,+ });}- const scrapeIds = data?.map(x => x.id) ?? [];- let scrapes: Document[] = [];- let iteratedOver = 0;- let bytes = 0;- const bytesLimit = 10485760; // 10 MiB in bytes-- const scrapeBlobs = await Promise.all(- scrapeIds.map(async x => [x, (await getJobFromGCS(x))?.[0]]),- );-- for (const [id, scrape] of scrapeBlobs) {- if (scrape) {- scrapes.push(scrape);- bytes += JSON.stringify(scrape).length;- } else {- logger.warn("Job was considered done, but returnvalue is undefined!", {- jobId: id,- returnvalue: scrape,- });- }-- iteratedOver++;-- if (bytes > bytesLimit) {- break;- }- }+ iteratedOver++;- if (bytes > bytesLimit && scrapes.length !== 1) {- scrapes.splice(scrapes.length - 1, 1);- iteratedOver--;+ if (bytes > bytesLimit) {+ break;}+ }- outputBulkB = {- data: scrapes,- next:- (outputBulkA.total ?? 0) > start + iteratedOver- ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`- : undefined,- };+ if (bytes > bytesLimit && scrapes.length !== 1) {+ scrapes.splice(scrapes.length - 1, 1);+ iteratedOver--;}+ outputBulkB = {+ data: scrapes,+ next:+ (outputBulkA.total ?? 0) > start + iteratedOver+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`+ : undefined,+ };+return res.status(200).json({success: true,status: outputBulkA.status ?? "scraping",diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex e83489826..5b5fdfa7a 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -17,6 +17,7 @@ import { _addScrapeJobToBullMQ } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { fromV1ScrapeOptions } from "../v2/types";import { checkPermissions } from "../../lib/permissions";+import { crawlGroup } from "../../services/worker/nuq";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -135,6 +136,12 @@ export async function crawlController(});}+ await crawlGroup.addGroup(+ id,+ sc.team_id,+ (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ );+await saveCrawl(id, sc);await markCrawlActive(id);diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 04a478bb1..418de7b7a 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -25,6 +25,7 @@ import { logger as _logger } from "../../lib/logger";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { checkPermissions } from "../../lib/permissions";+import { crawlGroup } from "../../services/worker/nuq";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -129,6 +130,11 @@ export async function batchScrapeController(};if (!req.body.appendToId) {+ await crawlGroup.addGroup(+ id,+ sc.team_id,+ (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ );await saveCrawl(id, sc);await markCrawlActive(id);}diff --git a/apps/api/src/controllers/v2/crawl-ongoing.ts b/apps/api/src/controllers/v2/crawl-ongoing.tsindex 74d8ae84e..a27771da3 100644--- a/apps/api/src/controllers/v2/crawl-ongoing.ts+++ b/apps/api/src/controllers/v2/crawl-ongoing.ts@@ -4,15 +4,18 @@ import {RequestWithAuth,toV2CrawlerOptions,} from "./types";-import { getCrawl, getCrawlsByTeamId } from "../../lib/crawl-redis";+import { getCrawl } from "../../lib/crawl-redis";import { configDotenv } from "dotenv";+import { crawlGroup } from "../../services/worker/nuq";configDotenv();export async function ongoingCrawlsController(req: RequestWithAuth<{}, undefined, OngoingCrawlsResponse>,res: Response<OngoingCrawlsResponse>,) {- const ids = await getCrawlsByTeamId(req.auth.team_id);+ const ids = (await crawlGroup.getOngoingByOwner(req.auth.team_id)).map(+ x => x.id,+ );const crawls = (await Promise.all(ids.map(async id => ({ ...(await getCrawl(id)), id })))diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 16e6d845c..551cf9095 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -21,7 +21,12 @@ import { configDotenv } from "dotenv";import { logger } from "../../lib/logger";import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getJobFromGCS } from "../../lib/gcs-jobs";-import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";+import {+ scrapeQueue,+ NuQJob,+ NuQJobStatus,+ crawlGroup,+} from "../../services/worker/nuq";import { ScrapeJobSingleUrls } from "../../types";import { redisEvictConnection } from "../../../src/services/redis";import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";@@ -177,298 +182,109 @@ export async function crawlStatusController(? start + parseInt(req.query.limit, 10) - 1: undefined;+ const group = await crawlGroup.getGroup(req.params.jobId);+ const groupAnyJob = await scrapeQueue.getGroupAnyJob(+ req.params.jobId,+ req.auth.team_id,+ );const sc = await getCrawl(req.params.jobId);- const djoCutoff = Date.now() - 250;-- if (sc) {- if (sc.team_id !== req.auth.team_id) {- // Allow preview tokens to access preview jobs regardless of IP-derived team_id mismatch- const scIsPreview = sc.team_id?.startsWith("preview");- if (!(isPreviewTeam && scIsPreview)) {- return res.status(404).json({ success: false, error: "Job not found" });- }- }- } else if (process.env.USE_DB_AUTHENTICATION === "true") {- if (isPreviewTeam) {- // Preview teams do not persist crawls in DB; avoid DB lookup that can 500- return res.status(404).json({ success: false, error: "Job not found" });- }- const { data: crawlJobs, error: crawlJobError } = await supabase_rr_service- .from("firecrawl_jobs")- .select("*")- .eq("job_id", req.params.jobId)- .limit(1);-- let crawlAdded: Date;- let crawlTeamId: string;-- if (crawlJobError || !crawlJobs || crawlJobs.length === 0) {- const { data: scrapeJobs, error: scrapeJobError } =- await supabase_rr_service- .from("firecrawl_jobs")- .select("*")- .eq("crawl_id", req.params.jobId)- .order("date_added", { ascending: false })- .limit(1);-- if (scrapeJobError || !scrapeJobs || scrapeJobs.length === 0) {- return res.status(404).json({ success: false, error: "Job not found" });- }-- const scrapeJob = scrapeJobs[0];- crawlAdded = new Date(scrapeJob.date_added); // use last scrape as date added for crawl- crawlTeamId = scrapeJob.team_id;- } else {- const crawlJob = crawlJobs[0];- crawlAdded = new Date(crawlJob.date_added);- crawlTeamId = crawlJob.team_id;- }+ if (!group || (!groupAnyJob && (!sc || sc.team_id !== req.auth.team_id))) {+ return res.status(404).json({ success: false, error: "Job not found" });+ }- if (crawlTeamId !== req.auth.team_id) {- return res.status(404).json({ success: false, error: "Job not found" });- }+ const zeroDataRetention = !!(+ groupAnyJob?.data?.zeroDataRetention ?? sc?.zeroDataRetention+ );- const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;- const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;+ const numericStats = await scrapeQueue.getGroupNumericStats(+ req.params.jobId,+ logger.child({ zeroDataRetention }),+ );- if (new Date().valueOf() - crawlAdded.valueOf() > crawlTtlMs) {- return res.status(404).json({ success: false, error: "Job expired" });- }- } else {- // if SC is gone and no DB, that means the job is expired, or never existed to begin with- return res.status(404).json({ success: false, error: "Job not found" });- }+ const creditsRpc =+ process.env.USE_DB_AUTHENTICATION === "true"+ ? await supabase_service.rpc(+ "credits_billed_by_crawl_id_1",+ {+ i_crawl_id: req.params.jobId,+ },+ { get: true },+ )+ : null;let outputBulkA: {status?: "completed" | "scraping" | "cancelled";completed?: number;total?: number;creditsUsed?: number;+ } = {+ status: group.status === "active" ? "scraping" : group.status,+ completed: numericStats.completed ?? 0,+ total:+ (numericStats.completed ?? 0) ++ (numericStats.active ?? 0) ++ (numericStats.queued ?? 0) ++ (numericStats.backlog ?? 0),+ creditsUsed: creditsRpc?.data?.[0]?.credits_billed ?? -1,};- if (sc) {- // Local mode (easier, low-pressure)- const kickoffFinished = await isCrawlKickoffFinished(req.params.jobId);- let total = await getCrawlQualifiedJobCount(req.params.jobId);-- let completed = await getDoneJobsOrderedLength(req.params.jobId, djoCutoff);- let creditsUsed =- completed *- (sc.scrapeOptions?.formats?.find(- x => typeof x === "object" && x.type === "json",- )- ? 5- : 1);-- if (process.env.USE_DB_AUTHENTICATION === "true" && !isPreviewTeam) {- const creditsRpc = await supabase_service.rpc(- "credits_billed_by_crawl_id_1",- {- i_crawl_id: req.params.jobId,- },- { get: true },- );-- creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? creditsUsed;-- if (- total === 0 &&- completed === 0 &&- Date.now() - sc.createdAt > 1000 * 60- ) {- const x = await supabase_service.rpc(- "crawl_status_job_count_1",- {- i_team_id: req.auth.team_id,- i_crawl_id: req.params.jobId,- },- { get: true },- );-- total = x.data?.[0]?.count ?? 0;- completed = total;- }- }-- outputBulkA = {- status: sc.cancelled- ? "cancelled"- : completed === total && kickoffFinished- ? "completed"- : "scraping",- total,- completed,- creditsUsed,- };- } else {- // DB must be specified at this point, otherwise control flow kills execution earlier- // DB mode (once job expires from Redis)- const { data: crawlJobCounts, error: crawlJobCountsError } =- await supabase_service.rpc(- "crawl_status_job_count_1",- {- i_team_id: req.auth.team_id,- i_crawl_id: req.params.jobId,- },- { get: true },- );-- if (crawlJobCountsError || !crawlJobCounts || crawlJobCounts.length === 0) {- logger.error("Error getting crawl job count", {- error: crawlJobCountsError,- });- throw new Error("Error getting crawl job count", {- cause: crawlJobCountsError,- });- }-- const crawlJobCount: number = crawlJobCounts[0].count ?? 0;-- const creditsRpc = await supabase_service.rpc(- "credits_billed_by_crawl_id_1",- {- i_crawl_id: req.params.jobId,- },- { get: true },- );-- const creditsUsed = creditsRpc.data?.[0]?.credits_billed ?? crawlJobCount;-- outputBulkA = {- status: "completed", // Salvage expired job- total: crawlJobCount,- completed: crawlJobCount,- creditsUsed,- };- }-let outputBulkB: {data: Document[];next: string | undefined;};- if (sc || process.env.USE_DB_AUTHENTICATION !== "true" || isPreviewTeam) {- const doneJobs = await getDoneJobsOrderedUntil(- req.params.jobId,- djoCutoff,- start,- end !== undefined ? end - start : 100,- );-- let scrapes: Document[] = [];- let iteratedOver = 0;- let bytes = 0;- const bytesLimit = 10485760; // 10 MiB in bytes-- for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) {- const jobIds = doneJobs.slice(i * 50, (i + 1) * 50);- const jobs = await getJobs(jobIds, logger);-- for (const job of jobs) {- if (job.status === "failed") {- continue;- } else {- if (job?.returnvalue) {- scrapes.push(job.returnvalue);- bytes += JSON.stringify(job.returnvalue).length;- } else {- logger.warn(- "Job was considered done, but returnvalue is undefined!",- {- scrapeId: job.id,- crawlId: req.params.jobId,- state: job.status,- returnvalue: job?.returnvalue,- },- );- }-- iteratedOver++;- }-- if (bytes > bytesLimit) {- break;- }- }-- if (bytes > bytesLimit) {- break;- }- }-- if (bytes > bytesLimit && scrapes.length !== 1) {- scrapes.splice(scrapes.length - 1, 1);- iteratedOver--;- }-- outputBulkB = {- data: scrapes,- next:- (outputBulkA.total ?? 0) > start + iteratedOver- ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`- : undefined,- };- } else {- // new DB-based path- const { data, error } = await supabase_service.rpc(- "crawl_status_1",- {- i_team_id: req.auth.team_id,- i_crawl_id: req.params.jobId,- i_start: start,- i_end: end ?? start + 100,- },- { get: true },- );-- if (error || !data) {- logger.error("Error getting crawl status from DB", { error });- throw new Error("Error getting crawl status from DB", { cause: error });+ const doneJobs = await scrapeQueue.getCrawlJobsForListing(+ req.params.jobId,+ end !== undefined ? end - start : 100,+ start,+ logger.child({ zeroDataRetention }),+ );++ let scrapes: Document[] = [];+ let iteratedOver = 0;+ let bytes = 0;+ const bytesLimit = 10485760; // 10 MiB in bytes++ const scrapeBlobs = await Promise.all(+ doneJobs.map(+ async x =>+ [x.id, x.returnvalue ?? (await getJobFromGCS(x.id))?.[0]] as const,+ ),+ );++ for (const [id, scrape] of scrapeBlobs) {+ if (scrape) {+ scrapes.push(scrape);+ bytes += JSON.stringify(scrape).length;+ } else {+ logger.warn("Job was considered done, but returnvalue is undefined!", {+ jobId: id,+ returnvalue: scrape,+ zeroDataRetention,+ });}- const scrapeIds = data?.map(x => x.id) ?? [];- let scrapes: Document[] = [];- let iteratedOver = 0;- let bytes = 0;- const bytesLimit = 10485760; // 10 MiB in bytes-- const scrapeBlobs = await Promise.all(- scrapeIds.map(async x => [x, (await getJobFromGCS(x))?.[0]]),- );-- for (const [id, scrape] of scrapeBlobs) {- if (scrape) {- scrapes.push(scrape);- bytes += JSON.stringify(scrape).length;- } else {- logger.warn("Job was considered done, but returnvalue is undefined!", {- jobId: id,- returnvalue: scrape,- });- }-- iteratedOver++;+ iteratedOver++;- if (bytes > bytesLimit) {- break;- }- }-- if (bytes > bytesLimit && scrapes.length !== 1) {- scrapes.splice(scrapes.length - 1, 1);- iteratedOver--;+ if (bytes > bytesLimit) {+ break;}+ }- outputBulkB = {- data: scrapes,- next:- (outputBulkA.total ?? 0) > start + iteratedOver- ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v2/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`- : undefined,- };+ if (bytes > bytesLimit && scrapes.length !== 1) {+ scrapes.splice(scrapes.length - 1, 1);+ iteratedOver--;}+ outputBulkB = {+ data: scrapes,+ next:+ (outputBulkA.total ?? 0) > start + iteratedOver+ ? `${process.env.ENV === "local" ? req.protocol : "https"}://${req.get("host")}/v1/${isBatch ? "batch/scrape" : "crawl"}/${req.params.jobId}?skip=${start + iteratedOver}${req.query.limit ? `&limit=${req.query.limit}` : ""}`+ : undefined,+ };+// Check for robots.txt blocked URLs and add warning if foundlet warning: string | undefined;try {@@ -484,11 +300,15 @@ export async function crawlStatusController(}} catch (error) {// If we can't check robots blocked URLs, continue without warning- logger.debug("Failed to check robots blocked URLs", { error });+ logger.debug("Failed to check robots blocked URLs", {+ error,+ zeroDataRetention,+ });}// Check if we should warn about base domain for crawl results- const resultCount = outputBulkA.completed ?? outputBulkA.total ?? outputBulkB.data.length;+ const resultCount =+ outputBulkA.completed ?? outputBulkA.total ?? outputBulkB.data.length;const currentStatus = outputBulkA.status ?? "scraping";if (!warning && currentStatus !== "scraping" && resultCount <= 1) {// Get the original crawl URL and options from stored crawl datadiff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 2a74b00fa..d588d3a00 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -20,6 +20,7 @@ import { CostTracking } from "../../lib/cost-tracking";import { checkPermissions } from "../../lib/permissions";import { buildPromptWithWebsiteStructure } from "../../lib/map-utils";import { modifyCrawlUrl } from "../../utils/url-utils";+import { crawlGroup } from "../../services/worker/nuq";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -202,6 +203,12 @@ export async function crawlController(});}+ await crawlGroup.addGroup(+ id,+ sc.team_id,+ (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ );+await saveCrawl(id, sc);await markCrawlActive(id);diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 122836720..513a1ea73 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -1,7 +1,7 @@import "dotenv/config";import { type ChildProcess, spawn } from "child_process";import * as net from "net";-import { basename } from "path";+import { basename, join } from "path";import { HTML_TO_MARKDOWN_PATH } from "./natives";import { createWriteStream } from "fs";@@ -11,6 +11,15 @@ const stopping = new WeakSet<ChildProcess>(); // processes we're intentionally slet IS_DEV = false;let restartSignal: AbortController | null = null;let shuttingDown = false;+let nuqPostgresContainer: {+ containerName: string;+ containerRuntime: string;+} | null = null;++// Get the monorepo root (apps/api/dist/src -> ../../../..)+// __dirname is available in CommonJS (which this compiles to)+const MONOREPO_ROOT = join(__dirname, "..", "..", "..", "..");+const NUQ_POSTGRES_PATH = join(MONOREPO_ROOT, "apps", "nuq-postgres");interface ProcessResult {promise: Promise<void>;@@ -25,6 +34,10 @@ interface Services {extractWorker?: ProcessResult;indexWorker?: ProcessResult;command?: ProcessResult;+ nuqPostgres?: {+ containerName: string;+ containerRuntime: string;+ };}const colors = {@@ -49,6 +62,8 @@ const processGroupColors: Record<string, string> = {index: colors.yellow,go: colors.yellow,command: colors.white,+ docker: colors.blue,+ podman: colors.blue,};function getProcessGroup(name: string): string {@@ -345,6 +360,174 @@ function terminateProcess(proc: ChildProcess, force: boolean): Promise<void> {});}+async function detectContainerRuntime(): Promise<string | null> {+ // Try docker first, then podman+ for (const runtime of ["docker", "podman"]) {+ try {+ const check = execForward(`${runtime}@check`, `${runtime} --version`);+ await check.promise;+ return runtime;+ } catch {+ // Runtime not available, try next+ }+ }+ return null;+}++async function isContainerRunning(+ runtime: string,+ containerName: string,+): Promise<boolean> {+ try {+ const check = execForward(+ `${runtime}@ps`,+ `${runtime} ps -a --filter name=^${containerName}$ --format '{{.Names}}'`,+ );+ await check.promise;+ return true;+ } catch {+ return false;+ }+}++async function stopAndRemoveContainer(+ runtime: string,+ containerName: string,+): Promise<void> {+ const isRunning = await isContainerRunning(runtime, containerName);+ if (isRunning) {+ logger.info(`Stopping existing container: ${containerName}`);+ try {+ const stop = execForward(+ `${runtime}@stop`,+ `${runtime} stop ${containerName}`,+ );+ await stop.promise;+ } catch (e) {+ logger.warn(`Failed to stop container ${containerName}, continuing...`);+ }+ }++ // Try to remove the container (whether it was running or not)+ try {+ const remove = execForward(+ `${runtime}@rm`,+ `${runtime} rm -f ${containerName}`,+ );+ await remove.promise;+ } catch (e) {+ // Container might not exist, that's fine+ }+}++async function buildNuqPostgresImage(runtime: string): Promise<void> {+ logger.info("Building nuq-postgres Docker image");+ const build = execForward(+ `${runtime}@build`,+ `${runtime} build -t firecrawl-nuq-postgres:latest ${NUQ_POSTGRES_PATH}`,+ );+ await build.promise;+ logger.success("nuq-postgres image built");+}++async function startNuqPostgresContainer(+ runtime: string,+ containerName: string,+): Promise<void> {+ logger.info(`Starting PostgreSQL container: ${containerName}`);+ const start = execForward(+ `${runtime}@start`,+ `${runtime} run -d --name ${containerName} -p 5432:5432 -e POSTGRES_PASSWORD=postgres -e POSTGRES_USER=postgres -e POSTGRES_DB=postgres firecrawl-nuq-postgres:latest`,+ );+ await start.promise;+ logger.success(`PostgreSQL container started: ${containerName}`);+}++async function waitForPostgres(+ host: string,+ port: number,+ timeoutMs: number = 30000,+): Promise<void> {+ logger.info("Waiting for PostgreSQL to be ready...");+ const start = Date.now();++ while (Date.now() - start < timeoutMs) {+ try {+ // Try to connect to PostgreSQL using a simple query+ const { Client } = await import("pg");+ const client = new Client({+ host,+ port,+ user: "postgres",+ password: "postgres",+ database: "postgres",+ connectionTimeoutMillis: 2000,+ });++ await client.connect();+ await client.query("SELECT 1");+ await client.end();++ logger.success("PostgreSQL is ready");+ return;+ } catch (e) {+ // Not ready yet, wait and retry+ await new Promise(resolve => setTimeout(resolve, 500));+ }+ }++ throw new Error(`PostgreSQL did not become ready within ${timeoutMs}ms`);+}++async function setupNuqPostgres(): Promise<Services["nuqPostgres"]> {+ if (process.env.NUQ_DATABASE_URL) {+ logger.info("NUQ_DATABASE_URL is set, skipping container management");+ return undefined;+ }++ logger.section("Setting up NUQ PostgreSQL container");++ const runtime = await detectContainerRuntime();+ if (!runtime) {+ throw new Error(+ "Neither Docker nor Podman found. Please install Docker/Podman or set NUQ_DATABASE_URL manually.",+ );+ }++ logger.success(`Using container runtime: ${runtime}`);++ const containerName = "firecrawl-nuq-postgres";++ // Stop and remove any existing container+ await stopAndRemoveContainer(runtime, containerName);++ // Build the image+ await buildNuqPostgresImage(runtime);++ // Start the container+ await startNuqPostgresContainer(runtime, containerName);++ // Wait for PostgreSQL to be ready+ await waitForPostgres("localhost", 5432);++ // Set environment variables for the services+ const dbUrl = "postgresql://postgres:postgres@localhost:5432/postgres";+ process.env.NUQ_DATABASE_URL = dbUrl;+ process.env.NUQ_DATABASE_URL_LISTEN = dbUrl;++ logger.success("NUQ PostgreSQL container is ready");++ const containerInfo = {+ containerName,+ containerRuntime: runtime,+ };++ // Store globally for graceful shutdown+ nuqPostgresContainer = containerInfo;++ return containerInfo;+}+async function installDependencies() {logger.section("Installing dependencies");@@ -388,6 +571,9 @@ async function installDependencies() {}async function startServices(command?: string[]): Promise<Services> {+ // Setup NUQ PostgreSQL container if needed+ const nuqPostgres = await setupNuqPostgres();+logger.section("Starting services");const api = execForward(@@ -492,6 +678,7 @@ async function startServices(command?: string[]): Promise<Services> {indexWorker,extractWorker,command: commandProcess,+ nuqPostgres,};}@@ -515,6 +702,16 @@ async function stopDevelopmentServices(services: Services) {logger.error("Error while stopping processes", e);}),]);++ // Stop and remove NUQ PostgreSQL container if it was started by harness+ if (services.nuqPostgres) {+ logger.info("Stopping NUQ PostgreSQL container");+ await stopAndRemoveContainer(+ services.nuqPostgres.containerRuntime,+ services.nuqPostgres.containerName,+ );+ logger.success("NUQ PostgreSQL container stopped");+ }}async function runDevMode(): Promise<void> {@@ -664,6 +861,18 @@ async function gracefulShutdown() {terminateProcess(proc, forceTerminate),);await Promise.allSettled(terminationPromises);++ // Stop and remove NUQ PostgreSQL container if it was started by harness+ if (nuqPostgresContainer) {+ logger.info("Stopping NUQ PostgreSQL container");+ await stopAndRemoveContainer(+ nuqPostgresContainer.containerRuntime,+ nuqPostgresContainer.containerName,+ );+ logger.success("NUQ PostgreSQL container stopped");+ nuqPostgresContainer = null;+ }+logger.success("All processes terminated");}diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex c0a435478..03a6ccdaf 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -20,6 +20,10 @@ import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { BullMQOtel } from "bullmq-otel";import { getErrorContactMessage } from "../lib/deployment";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";+import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";+import { getCrawl } from "../lib/crawl-redis";+import type { Logger } from "winston";+import { finishCrawlSuper } from "./worker/crawl-logic";configDotenv();@@ -249,6 +253,117 @@ const workerFun = async (}};+async function processFinishCrawlJobInternal(job: NuQJob, logger: Logger) {+ if (!job.groupId) {+ throw new Error("crawlFinish job with no groupId");+ }++ if (!job.ownerId) {+ throw new Error("crawlFinish job with no ownerId");+ }++ const sc = await getCrawl(job.groupId);++ if (!sc) {+ throw new Error("crawlFinish job with sc expired");+ }++ const anyJob = await scrapeQueue.getGroupAnyJob(job.groupId, job.ownerId);++ if (!anyJob) {+ throw new Error("crawlFinish couldn't find anyJob");+ }++ await finishCrawlSuper(anyJob);+}++const crawlFinishWorker = async () => {+ const __logger = _logger.child({+ module: "extract-worker",+ method: "crawlFinishWorker",+ });++ let noJobTimeout = 1500;++ while (!isShuttingDown) {+ const job = await crawlFinishedQueue.getJobToProcess();++ if (job === null) {+ __logger.info("No jobs to process", { module: "nuq/metrics" });+ await new Promise(resolve => setTimeout(resolve, noJobTimeout));+ if (!process.env.NUQ_RABBITMQ_URL) {+ noJobTimeout = Math.min(noJobTimeout * 2, 10000);+ }+ continue;+ }++ noJobTimeout = 500;++ const logger = __logger.child({+ zeroDataRetention: job.data?.zeroDataRetention ?? false,+ crawlId: job.groupId,+ });++ logger.info("Acquired job");++ const lockRenewInterval = setInterval(async () => {+ logger.info("Renewing lock");+ if (!(await crawlFinishedQueue.renewLock(job.id, job.lock!, logger))) {+ logger.warn("Failed to renew lock");+ clearInterval(lockRenewInterval);+ return;+ }+ logger.info("Renewed lock");+ }, 15000);++ let processResult:+ | {+ ok: true;+ data: Awaited<ReturnType<typeof processFinishCrawlJobInternal>>;+ }+ | { ok: false; error: any };++ try {+ processResult = {+ ok: true,+ data: await processFinishCrawlJobInternal(job, logger),+ };+ } catch (error) {+ processResult = { ok: false, error };+ }++ clearInterval(lockRenewInterval);++ if (processResult.ok) {+ if (+ !(await crawlFinishedQueue.jobFinish(+ job.id,+ job.lock!,+ processResult.data,+ logger,+ ))+ ) {+ logger.warn("Could not update job status");+ }+ } else {+ if (+ !(await crawlFinishedQueue.jobFail(+ job.id,+ job.lock!,+ processResult.error instanceof Error+ ? processResult.error.message+ : typeof processResult.error === "string"+ ? processResult.error+ : JSON.stringify(processResult.error),+ logger,+ ))+ ) {+ logger.warn("Could not update job status");+ }+ }+ }+};+// Start all workersconst app = Express();@@ -297,7 +412,10 @@ app.listen(workerPort, () => {process.exit(1);});- await Promise.all([workerFun(getExtractQueue(), processExtractJobInternal)]);+ await Promise.all([+ workerFun(getExtractQueue(), processExtractJobInternal),+ crawlFinishWorker(),+ ]);_logger.info("All workers exited. Waiting for all jobs to finish...");diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex e662e999b..b78be69f6 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -37,6 +37,8 @@ import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";import { _addScrapeJobToBullMQ } from "../queue-jobs";import { BullMQOtel } from "bullmq-otel";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";+import { crawlGroup } from "../worker/nuq";+import { getACUCTeam } from "../../controllers/auth";const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;const workerStalledCheckInterval =@@ -490,6 +492,15 @@ const processPrecrawlJob = async (token: string, job: Job) => {// });// }+ await crawlGroup.addGroup(+ crawlId,+ sc.team_id,+ ((await getACUCTeam(sc.team_id))?.flags?.crawlTtlHours ?? 24) *+ 60 *+ 60 *+ 1000,+ );+await saveCrawl(crawlId, sc);await _addScrapeJobToBullMQ(diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex af3489d44..eda2398f7 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -1,320 +1,179 @@import { logger as _logger } from "../../lib/logger";import {- addCrawlJobs,finishCrawl,getCrawlJobs,getDoneJobsOrderedLength,- lockURLsIndividually,- StoredCrawl,- unPreFinishCrawl,} from "../../lib/crawl-redis";-import { redisEvictConnection } from "../redis";import { getCrawl } from "../../lib/crawl-redis";-import { finishCrawlPre } from "../../lib/crawl-redis";-import { getACUCTeam } from "../../controllers/auth";-import { crawlToCrawler } from "../../lib/crawl-redis";import { supabase_service } from "../supabase";-import { v4 as uuidv4 } from "uuid";-import { addScrapeJobs } from "../queue-jobs";import { getJobs } from "../../controllers/v1/crawl-status";import { logJob } from "../logging/log_job";-import { createWebhookSender, WebhookEvent } from "../webhook";-import { hasFormatOfType } from "../../lib/format-utils";+import { createWebhookSender, WebhookEvent } from "../webhook/index";import type { NuQJob } from "./nuq";-import { ScrapeJobData } from "../../types";-export async function finishCrawlIfNeeded(- job: NuQJob<ScrapeJobData>,- sc: StoredCrawl,-) {- const crawlId = job.data.crawl_id!;+export async function finishCrawlSuper(job: NuQJob<any>) {+ const crawlId = job.groupId;if (!crawlId) {return;}+ const sc = await getCrawl(crawlId);++ if (!sc) {+ return;+ }+const logger = _logger.child({module: "queue-worker",- method: "finishCrawlIfNeeded",+ method: "finishCrawl",jobId: job.id,scrapeId: job.id,crawlId,zeroDataRetention: sc.internalOptions.zeroDataRetention,});- if (await finishCrawlPre(crawlId, logger)) {- logger.info("Crawl is pre-finished, checking if we need to add more jobs");- if (- sc.crawlerOptions &&- !(await redisEvictConnection.exists(- "crawl:" + crawlId + ":invisible_urls",- ))- ) {- await redisEvictConnection.set(- "crawl:" + crawlId + ":invisible_urls",- "done",- "EX",- 60 * 60 * 24,- );-- const sc = (await getCrawl(crawlId))!;-- const visitedUrls = new Set(- await redisEvictConnection.smembers(- "crawl:" + crawlId + ":visited_unique",- ),- );-- logger.info("Visited URLs", {- visitedUrls: visitedUrls.size,- });-- let lastUrls: string[] = [];- const useDbAuthentication = process.env.USE_DB_AUTHENTICATION === "true";- if (- useDbAuthentication &&- hasFormatOfType(sc.scrapeOptions.formats, "changeTracking")- ) {- lastUrls = (- (- await supabase_service.rpc("diff_get_last_crawl_urls", {- i_team_id: job.data.team_id,- i_url: sc.originUrl!,- })- ).data ?? []- ).map(x => x.url);- }-- const lastUrlsSet = new Set(lastUrls);-- logger.info("Last URLs", {- lastUrls: lastUrlsSet.size,+ logger.info("Finishing crawl");+ await finishCrawl(crawlId, logger);++ if (!job.data.v1) {+ const jobIDs = await getCrawlJobs(crawlId);++ const jobs = (await getJobs(jobIDs)).sort(+ (a, b) => a.timestamp - b.timestamp,+ );+ // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));+ const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")+ ? "failed"+ : "completed";++ const fullDocs = jobs+ .map(x =>+ x.returnvalue+ ? Array.isArray(x.returnvalue)+ ? x.returnvalue[0]+ : x.returnvalue+ : null,+ )+ .filter(x => x !== null);++ await logJob(+ {+ job_id: crawlId,+ success: jobStatus === "completed",+ message: sc.cancelled ? "Cancelled" : undefined,+ num_docs: fullDocs.length,+ docs: [],+ time_taken: (Date.now() - sc.createdAt) / 1000,+ team_id: job.data.team_id,+ mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",+ url: sc.originUrl!,+ scrapeOptions: sc.scrapeOptions,+ crawlerOptions: sc.crawlerOptions,+ origin: sc.originUrl!,+ integration: job.data.integration,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ false,+ sc.internalOptions?.bypassBilling ?? false,+ );++ // v0 web hooks, call when done with all the data+ if (!job.data.v1) {+ const sender = await createWebhookSender({+ teamId: job.data.team_id,+ jobId: crawlId,+ webhook: job.data.webhook,+ v0: true,});-- const crawler = crawlToCrawler(- crawlId,- sc,- (await getACUCTeam(job.data.team_id!))?.flags ?? null,- sc.originUrl!,- sc.crawlerOptions,- );-- const univistedUrls = await crawler.filterLinks(- Array.from(lastUrlsSet).filter(x => !visitedUrls.has(x)),- Infinity,- sc.crawlerOptions.maxDepth ?? 10,- );-- const addableJobCount =- sc.crawlerOptions.limit === undefined- ? Infinity- : sc.crawlerOptions.limit - (await getDoneJobsOrderedLength(crawlId));-- if (univistedUrls.links.length !== 0 && addableJobCount > 0) {- logger.info("Adding jobs", {- univistedUrls: univistedUrls.links.length,- addableJobCount,- });-- const jobs = univistedUrls.links.slice(0, addableJobCount).map(url => {- const uuid = uuidv4();- return {- jobId: uuid,- data: {- url,- mode: "single_urls" as const,- team_id: job.data.team_id,- crawlerOptions: {- ...sc.crawlerOptions,- urlInvisibleInCurrentCrawl: true,- },- scrapeOptions: sc.scrapeOptions,- internalOptions: sc.internalOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: crawlId,- sitemapped: true,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- apiKeyId: job.data.apiKeyId,- },- priority: 20, // TODO: make this dynamic- };- });-- const lockedIds = await lockURLsIndividually(- crawlId,- sc,- jobs.map(x => ({ id: x.jobId, url: x.data.url })),- );- const lockedJobs = jobs.filter(x =>- lockedIds.find(y => y.id === x.jobId),- );- await addCrawlJobs(- crawlId,- lockedJobs.map(x => x.jobId),- logger,- );- await addScrapeJobs(lockedJobs);-- if (lockedJobs.length > 0) {- logger.info("Added jobs, not going for the full finish", {- lockedJobs: lockedJobs.length,+ if (sender) {+ const documents = fullDocs.map((doc: any) => ({+ content: {+ content: doc?.content ?? doc?.rawHtml ?? doc?.markdown ?? "",+ markdown: doc?.markdown,+ metadata: doc?.metadata ?? {},+ },+ source: doc?.metadata?.sourceURL ?? doc?.url ?? "",+ }));+ if (sc.crawlerOptions !== null) {+ sender.send(WebhookEvent.CRAWL_COMPLETED, {+ success: true,+ data: documents,});-- await unPreFinishCrawl(crawlId);- return;} else {- logger.info(- "No jobs added (all discovered URLs were locked), finishing crawl",- );+ sender.send(WebhookEvent.BATCH_SCRAPE_COMPLETED, {+ success: true,+ data: documents,+ });}}}+ } else {+ const num_docs = await getDoneJobsOrderedLength(crawlId);+ const jobStatus = sc.cancelled ? "failed" : "completed";- logger.info("Finishing crawl");- await finishCrawl(crawlId, logger);-- if (!job.data.v1) {- const jobIDs = await getCrawlJobs(crawlId);-- const jobs = (await getJobs(jobIDs)).sort(- (a, b) => a.timestamp - b.timestamp,- );- // const jobStatuses = await Promise.all(jobs.map((x) => x.getState()));- const jobStatus = sc.cancelled // || jobStatuses.some((x) => x === "failed")- ? "failed"- : "completed";+ let credits_billed = null;- const fullDocs = jobs- .map(x =>- x.returnvalue- ? Array.isArray(x.returnvalue)- ? x.returnvalue[0]- : x.returnvalue- : null,- )- .filter(x => x !== null);-- await logJob(+ if (process.env.USE_DB_AUTHENTICATION === "true") {+ const creditsRpc = await supabase_service.rpc(+ "credits_billed_by_crawl_id_1",{- job_id: crawlId,- success: jobStatus === "completed",- message: sc.cancelled ? "Cancelled" : undefined,- num_docs: fullDocs.length,- docs: [],- time_taken: (Date.now() - sc.createdAt) / 1000,- team_id: job.data.team_id,- mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",- url: sc.originUrl!,- scrapeOptions: sc.scrapeOptions,- crawlerOptions: sc.crawlerOptions,- origin: sc.originUrl!,- integration: job.data.integration,- zeroDataRetention: job.data.zeroDataRetention,+ i_crawl_id: crawlId,},- false,- sc.internalOptions?.bypassBilling ?? false,);- // v0 web hooks, call when done with all the data- if (!job.data.v1) {- const sender = await createWebhookSender({- teamId: job.data.team_id,- jobId: crawlId,- webhook: job.data.webhook,- v0: true,+ credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;++ if (credits_billed === null) {+ logger.warn("Credits billed is null", {+ error: creditsRpc.error,});- if (sender) {- const documents = fullDocs.map((doc: any) => ({- content: {- content: doc?.content ?? doc?.rawHtml ?? doc?.markdown ?? "",- markdown: doc?.markdown,- metadata: doc?.metadata ?? {},- },- source: doc?.metadata?.sourceURL ?? doc?.url ?? "",- }));- if (sc.crawlerOptions !== null) {- sender.send(WebhookEvent.CRAWL_COMPLETED, {- success: true,- data: documents,- });- } else {- sender.send(WebhookEvent.BATCH_SCRAPE_COMPLETED, {- success: true,- data: documents,- });- }- }}- } else {- const num_docs = await getDoneJobsOrderedLength(crawlId);- const jobStatus = sc.cancelled ? "failed" : "completed";-- let credits_billed = null;-- if (process.env.USE_DB_AUTHENTICATION === "true") {- const creditsRpc = await supabase_service.rpc(- "credits_billed_by_crawl_id_1",- {- i_crawl_id: crawlId,- },- );-- credits_billed = creditsRpc.data?.[0]?.credits_billed ?? null;+ }- if (credits_billed === null) {- logger.warn("Credits billed is null", {- error: creditsRpc.error,+ await logJob(+ {+ job_id: crawlId,+ success: jobStatus === "completed",+ message: sc.cancelled ? "Cancelled" : undefined,+ num_docs,+ docs: [],+ time_taken: (Date.now() - sc.createdAt) / 1000,+ team_id: sc.team_id,+ scrapeOptions: sc.scrapeOptions,+ mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",+ url:+ sc?.originUrl ??+ (sc.crawlerOptions === null ? "Batch Scrape" : "Unknown"),+ crawlerOptions: sc.crawlerOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ credits_billed,+ zeroDataRetention: job.data.zeroDataRetention,+ },+ true,+ sc.internalOptions?.bypassBilling ?? false,+ );++ // v1 web hooks, call when done with no data, but with event completed+ if (job.data.v1 && job.data.webhook) {+ const sender = await createWebhookSender({+ teamId: job.data.team_id,+ jobId: crawlId,+ webhook: job.data.webhook,+ v0: false,+ });+ if (sender) {+ if (sc.crawlerOptions !== null) {+ sender.send(WebhookEvent.CRAWL_COMPLETED, {+ success: true,+ data: [],+ });+ } else {+ sender.send(WebhookEvent.BATCH_SCRAPE_COMPLETED, {+ success: true,+ data: [],});- }- }-- await logJob(- {- job_id: crawlId,- success: jobStatus === "completed",- message: sc.cancelled ? "Cancelled" : undefined,- num_docs,- docs: [],- time_taken: (Date.now() - sc.createdAt) / 1000,- team_id: sc.team_id,- scrapeOptions: sc.scrapeOptions,- mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",- url:- sc?.originUrl ??- (sc.crawlerOptions === null ? "Batch Scrape" : "Unknown"),- crawlerOptions: sc.crawlerOptions,- origin: job.data.origin,- integration: job.data.integration,- credits_billed,- zeroDataRetention: job.data.zeroDataRetention,- },- true,- sc.internalOptions?.bypassBilling ?? false,- );-- // v1 web hooks, call when done with no data, but with event completed- if (job.data.v1 && job.data.webhook) {- const sender = await createWebhookSender({- teamId: job.data.team_id,- jobId: crawlId,- webhook: job.data.webhook,- v0: false,- });- if (sender) {- if (sc.crawlerOptions !== null) {- sender.send(WebhookEvent.CRAWL_COMPLETED, {- success: true,- data: [],- });- } else {- sender.send(WebhookEvent.BATCH_SCRAPE_COMPLETED, {- success: true,- data: [],- });- }}}}diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 8c5126cd9..a0463dd2d 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -499,6 +499,95 @@ class NuQ<JobData = any, JobReturnValue = any> {}}+ public async getGroupAnyJob(+ groupId: string,+ ownerId: string,+ ): Promise<NuQJob<JobData, JobReturnValue> | null> {+ return this.rowToJob(+ (+ await nuqPool.query(+ `+ SELECT ${this.jobReturning.join(", ")}+ FROM ${this.queueName}+ WHERE ${this.queueName}.group_id = $1+ AND ${this.queueName}.owner_id = $2+ AND ${this.queueName}.data->>'mode' = 'single_urls'+ LIMIT 1;+ `,+ [groupId, normalizeOwnerId(ownerId)],+ )+ ).rows[0],+ );+ }++ public async getGroupNumericStats(+ groupId: string,+ _logger: Logger = logger,+ ): Promise<Record<NuQJobStatus, number>> {+ const start = Date.now();+ try {+ return Object.fromEntries(+ (+ await nuqPool.query(+ `+ SELECT ${this.queueName}.status::text as status, COUNT(*) as count+ FROM ${this.queueName}+ WHERE ${this.queueName}.group_id = $1+ AND ${this.queueName}.data->>'mode' = 'single_urls'+ GROUP BY ${this.queueName}.status+ UNION ALL+ SELECT 'backlog'::text as status, COUNT(*) as count+ FROM ${this.queueName}_backlog+ WHERE ${this.queueName}_backlog.group_id = $1+ AND ${this.queueName}_backlog.data->>'mode' = 'single_urls'+ `,+ [groupId],+ )+ ).rows.map(row => [row.status, parseInt(row.count, 10)]),+ );+ } finally {+ _logger.info("nuqGetGroupNumericStats metrics", {+ module: "nuq/metrics",+ method: "nuqGetGroupNumericStats",+ duration: Date.now() - start,+ crawlId: groupId,+ });+ }+ }++ // TODO: make more generalizable+ public async getCrawlJobsForListing(+ groupId: string,+ limit: number,+ offset: number,+ _logger: Logger = logger,+ ): Promise<NuQJob<JobData, JobReturnValue>[]> {+ const start = Date.now();+ try {+ return (+ await nuqPool.query(+ `+ SELECT ${this.jobReturning.join(", ")}+ FROM ${this.queueName}+ WHERE ${this.queueName}.group_id = $1+ AND ${this.queueName}.status = 'completed'+ AND ${this.queueName}.data->>'mode' = 'single_urls'+ ORDER BY finished_at ASC, created_at ASC+ LIMIT $2 OFFSET $3;+ `,+ [groupId, limit, offset],+ )+ ).rows.map(row => this.rowToJob(row)!);+ } finally {+ _logger.info("nuqGetCrawlJobsForListing metrics", {+ module: "nuq/metrics",+ method: "nuqGetCrawlJobsForListing",+ duration: Date.now() - start,+ crawlId: groupId,+ });+ }+ }+public async removeJob(id: string,_logger: Logger = logger,@@ -1190,6 +1279,169 @@ class NuQ<JobData = any, JobReturnValue = any> {}}+// === Group Management++export type NuQGroupStatus = "active" | "completed" | "cancelled";++export type NuQJobGroupInstance = {+ id: string;+ status: NuQGroupStatus;+ createdAt: Date;+ ownerId: string;+ ttl: number;+ expiresAt?: Date;+};++class NuQJobGroup {+ constructor(public readonly groupName: string) {}++ private readonly groupReturning = [+ "id",+ "status",+ "created_at",+ "owner_id",+ "ttl",+ "expires_at",+ ];++ private rowToGroup(row: any): NuQJobGroupInstance | null {+ if (!row) return null;+ return {+ id: row.id,+ status: row.status,+ createdAt: new Date(row.created_at),+ ownerId: row.owner_id,+ ttl: row.ttl,+ expiresAt: row.expires_at ? new Date(row.expires_at) : undefined,+ };+ }++ public async addGroup(+ id: string,+ ownerId: string,+ ttl?: number,+ _logger: Logger = logger,+ ): Promise<NuQJobGroupInstance> {+ return withSpan("nuq.addGroup", async span => {+ setSpanAttributes(span, {+ "nuq.group_name": this.groupName,+ "nuq.group_id": id,+ "nuq.ttl": ttl ?? 86400000,+ });++ const start = Date.now();+ try {+ const result = this.rowToGroup(+ (+ await nuqPool.query(+ `INSERT INTO ${this.groupName} (id, owner_id, ttl) VALUES ($1, $2, $3) RETURNING ${this.groupReturning.join(", ")};`,+ [id, normalizeOwnerId(ownerId), ttl ?? 86400000],+ )+ ).rows[0],+ )!;++ setSpanAttributes(span, {+ "nuq.group_created": true,+ });++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ _logger.info("nuqAddGroup metrics", {+ module: "nuq/metrics",+ method: "nuqAddGroup",+ duration,+ groupId: id,+ });+ }+ });+ }++ public async getGroup(+ id: string,+ _logger: Logger = logger,+ ): Promise<NuQJobGroupInstance | null> {+ return withSpan("nuq.getGroup", async span => {+ setSpanAttributes(span, {+ "nuq.group_name": this.groupName,+ "nuq.group_id": id,+ });++ const start = Date.now();+ try {+ const result = this.rowToGroup(+ (+ await nuqPool.query(+ `SELECT ${this.groupReturning.join(", ")} FROM ${this.groupName} WHERE ${this.groupName}.id = $1;`,+ [id],+ )+ ).rows[0],+ );++ setSpanAttributes(span, {+ "nuq.group_found": result !== null,+ "nuq.group_status": result?.status,+ });++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ _logger.info("nuqGetGroup metrics", {+ module: "nuq/metrics",+ method: "nuqGetGroup",+ duration,+ groupId: id,+ });+ }+ });+ }++ public async getOngoingByOwner(+ ownerId: string,+ _logger: Logger = logger,+ ): Promise<NuQJobGroupInstance[]> {+ return withSpan("nuq.getGroupByOwner", async span => {+ setSpanAttributes(span, {+ "nuq.group_name": this.groupName,+ "nuq.owner_id": ownerId,+ });++ const start = Date.now();+ try {+ const result = (+ await nuqPool.query(+ `SELECT ${this.groupReturning.join(", ")} FROM ${this.groupName} WHERE ${this.groupName}.owner_id = $1 AND ${this.groupName}.status = 'active'`,+ [normalizeOwnerId(ownerId)],+ )+ ).rows.map(x => this.rowToGroup(x)!);++ setSpanAttributes(span, {+ "nuq.groups_found": result.length,+ });++ return result;+ } finally {+ const duration = Date.now() - start;+ setSpanAttributes(span, {+ "nuq.duration_ms": duration,+ });+ _logger.info("nuqGetGroup metrics", {+ module: "nuq/metrics",+ method: "nuqGetGroupByOwner",+ duration,+ ownerId: ownerId,+ });+ }+ });+ }+}+export function nuqGetLocalMetrics(): string {return `# HELP nuq_pool_waiting_count Number of requests waiting in the pool\n# TYPE nuq_pool_waiting_count gauge\nnuq_pool_waiting_count ${nuqPool.waitingCount}\n# HELP nuq_pool_idle_count Number of connections idle in the pool\n# TYPE nuq_pool_idle_count gauge\nnuq_pool_idle_count ${nuqPool.idleCount}\n@@ -1214,6 +1466,9 @@ export async function nuqHealthCheck(): Promise<boolean> {export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape", {backlog: true,});+export const crawlFinishedQueue = new NuQ("nuq.queue_crawl_finished", {});++export const crawlGroup = new NuQJobGroup("nuq.group_crawl");// === Cleanupdiff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex 0d4804b82..db1b40f56 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -50,7 +50,6 @@ import { WebCrawler } from "../../scraper/WebScraper/crawler";import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { getBillingQueue } from "../queue-service";import type { Logger } from "winston";-import { finishCrawlIfNeeded } from "./crawl-logic";import {RacedRedirectError,ScrapeJobTimeoutError,@@ -491,8 +490,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {logger.debug("Declaring job as done...");await addCrawlJobDone(job.data.crawl_id, job.id, true, logger);-- await finishCrawlIfNeeded(job, sc);} else {try {signal?.throwIfAborted();@@ -569,8 +566,6 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {"crawl:" + job.data.crawl_id + ":jobs_qualified",job.id,);-- await finishCrawlIfNeeded(job, sc);}const isEarlyTimeout = error instanceof ScrapeJobTimeoutError;@@ -937,16 +932,12 @@ async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {logger.debug("Done queueing jobs!");await finishCrawlKickoff(job.data.crawl_id);- await finishCrawlIfNeeded(job, sc);return { success: true };} catch (error) {logger.error("An error occurred!", { error });await finishCrawlKickoff(job.data.crawl_id);const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;- if (sc) {- await finishCrawlIfNeeded(job, sc);- }return { success: false, error };}}@@ -1066,10 +1057,6 @@ async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {"crawl:" + job.data.crawl_id + ":sitemap_jobs_done",24 * 60 * 60,);-- if (sc) {- await finishCrawlIfNeeded(job, sc);- }}}diff --git a/apps/nuq-postgres/nuq.sql b/apps/nuq-postgres/nuq.sqlindex 0252dec84..119ec8d27 100644--- a/apps/nuq-postgres/nuq.sql+++ b/apps/nuq-postgres/nuq.sql@@ -9,6 +9,12 @@ EXCEPTIONWHEN duplicate_object THEN null;END $$;+DO $$ BEGIN+ CREATE TYPE nuq.group_status AS ENUM ('active', 'completed', 'cancelled');+EXCEPTION+ WHEN duplicate_object THEN null;+END $$;+CREATE TABLE IF NOT EXISTS nuq.queue_scrape (id uuid NOT NULL DEFAULT gen_random_uuid(),status nuq.job_status NOT NULL DEFAULT 'queued'::nuq.job_status,@@ -51,11 +57,11 @@ CREATE TABLE IF NOT EXISTS nuq.queue_scrape_backlog ();SELECT cron.schedule('nuq_queue_scrape_clean_completed', '*/5 * * * *', $$- DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour';+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'completed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '1 hour' AND group_id IS NULL;$$);SELECT cron.schedule('nuq_queue_scrape_clean_failed', '*/5 * * * *', $$- DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '6 hours';+ DELETE FROM nuq.queue_scrape WHERE nuq.queue_scrape.status = 'failed'::nuq.job_status AND nuq.queue_scrape.created_at < now() - interval '6 hours' AND group_id IS NULL;$$);SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$@@ -64,6 +70,107 @@ SELECT cron.schedule('nuq_queue_scrape_lock_reaper', '15 seconds', $$SELECT pg_notify('nuq.queue_scrape', (id::text || '|' || 'failed'::text)) FROM stallfail;$$);+SELECT cron.schedule('nuq_queue_scrape_backlog_reaper', '* * * * *', $$+ DELETE FROM nuq.queue_scrape_backlog+ WHERE nuq.queue_scrape_backlog.times_out_at < now();+$$);+SELECT cron.schedule('nuq_queue_scrape_reindex', '0 9 * * *', $$REINDEX TABLE CONCURRENTLY nuq.queue_scrape;$$);++CREATE TABLE IF NOT EXISTS nuq.queue_crawl_finished (+ id uuid NOT NULL DEFAULT gen_random_uuid(),+ status nuq.job_status NOT NULL DEFAULT 'queued'::nuq.job_status,+ data jsonb,+ created_at timestamp with time zone NOT NULL DEFAULT now(),+ priority int NOT NULL DEFAULT 0,+ lock uuid,+ locked_at timestamp with time zone,+ stalls integer,+ finished_at timestamp with time zone,+ listen_channel_id text, -- for listenable jobs over rabbitmq+ returnvalue jsonb, -- only for selfhost+ failedreason text, -- only for selfhost+ owner_id uuid,+ group_id uuid,+ CONSTRAINT queue_crawl_finished_pkey PRIMARY KEY (id)+);++ALTER TABLE nuq.queue_crawl_finished+SET (autovacuum_vacuum_scale_factor = 0.01,+ autovacuum_analyze_scale_factor = 0.01,+ autovacuum_vacuum_cost_limit = 2000,+ autovacuum_vacuum_cost_delay = 2);++CREATE INDEX IF NOT EXISTS queue_crawl_finished_active_locked_at_idx ON nuq.queue_crawl_finished USING btree (locked_at) WHERE (status = 'active'::nuq.job_status);+CREATE INDEX IF NOT EXISTS nuq_queue_crawl_finished_queued_optimal_2_idx ON nuq.queue_crawl_finished (priority ASC, created_at ASC, id) WHERE (status = 'queued'::nuq.job_status);+CREATE INDEX IF NOT EXISTS nuq_queue_crawl_finished_failed_created_at_idx ON nuq.queue_crawl_finished USING btree (created_at) WHERE (status = 'failed'::nuq.job_status);+CREATE INDEX IF NOT EXISTS nuq_queue_crawl_finished_completed_created_at_idx ON nuq.queue_crawl_finished USING btree (created_at) WHERE (status = 'completed'::nuq.job_status);++SELECT cron.schedule('nuq_queue_crawl_finished_clean_completed', '*/5 * * * *', $$+ DELETE FROM nuq.queue_crawl_finished WHERE nuq.queue_crawl_finished.status = 'completed'::nuq.job_status AND nuq.queue_crawl_finished.created_at < now() - interval '1 hour' AND group_id IS NULL;+$$);++SELECT cron.schedule('nuq_queue_crawl_finished_clean_failed', '*/5 * * * *', $$+ DELETE FROM nuq.queue_crawl_finished WHERE nuq.queue_crawl_finished.status = 'failed'::nuq.job_status AND nuq.queue_crawl_finished.created_at < now() - interval '6 hours' AND group_id IS NULL;+$$);++SELECT cron.schedule('nuq_queue_crawl_finished_lock_reaper', '15 seconds', $$+ UPDATE nuq.queue_crawl_finished SET status = 'queued'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_crawl_finished.locked_at <= now() - interval '1 minute' AND nuq.queue_crawl_finished.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_crawl_finished.stalls, 0) < 9;+ WITH stallfail AS (UPDATE nuq.queue_crawl_finished SET status = 'failed'::nuq.job_status, lock = null, locked_at = null, stalls = COALESCE(stalls, 0) + 1 WHERE nuq.queue_crawl_finished.locked_at <= now() - interval '1 minute' AND nuq.queue_crawl_finished.status = 'active'::nuq.job_status AND COALESCE(nuq.queue_crawl_finished.stalls, 0) >= 9 RETURNING id)+ SELECT pg_notify('nuq.queue_crawl_finished', (id::text || '|' || 'failed'::text)) FROM stallfail;+$$);++SELECT cron.schedule('nuq_queue_crawl_finished_reindex', '0 9 * * *', $$+ REINDEX TABLE CONCURRENTLY nuq.queue_crawl_finished;+$$);++CREATE TABLE IF NOT EXISTS nuq.group_crawl (+ id uuid NOT NULL,+ status nuq.group_status NOT NULL DEFAULT 'active'::nuq.group_status,+ created_at timestamptz NOT NULL DEFAULT now(),+ owner_id uuid NOT NULL,+ ttl int8 NOT NULL DEFAULT 86400000,+ expires_at timestamptz,+ CONSTRAINT group_crawl_pkey PRIMARY KEY (id)+);++SELECT cron.schedule('nuq_group_crawl_finished', '15 seconds', $$+ WITH finished_groups AS (+ UPDATE nuq.group_crawl+ SET status = 'completed'::nuq.group_status, expires_at = now() + MAKE_INTERVAL(secs => nuq.group_crawl.ttl / 1000)+ WHERE nuq.group_crawl.status = 'active'::nuq.group_status+ AND NOT EXISTS (+ SELECT 1 FROM nuq.queue_scrape+ WHERE nuq.queue_scrape.status IN ('active', 'queued')+ AND nuq.queue_scrape.group_id = nuq.group_crawl.id+ )+ AND NOT EXISTS (+ SELECT 1 FROM nuq.queue_scrape_backlog+ WHERE nuq.queue_scrape_backlog.group_id = nuq.group_crawl.id+ )+ RETURNING id, owner_id+ )+ INSERT INTO nuq.queue_crawl_finished (data, owner_id, group_id)+ SELECT '{}'::jsonb, finished_groups.owner_id, finished_groups.id+ FROM finished_groups;+$$);++SELECT cron.schedule('nuq_group_crawl_clean', '*/5 * * * *', $$+ WITH cleaned_groups AS (+ DELETE FROM nuq.group_crawl+ WHERE nuq.group_crawl.status = 'completed'::nuq.group_status+ AND nuq.group_crawl.expires_at < now();+ RETURNING *;+ ), cleaned_jobs_queue_scrape AS (+ DELETE FROM nuq.queue_scrape+ WHERE nuq.queue_scrape.group_id IN (SELECT id FROM cleaned_groups)+ ), cleaned_jobs_queue_scrape_backlog AS (+ DELETE FROM nuq.queue_scrape_backlog+ WHERE nuq.queue_scrape_backlog.group_id IN (SELECT id FROM cleaned_groups)+ ), cleaned_jobs_crawl_finished AS (+ DELETE FROM nuq.queue_crawl_finished+ WHERE nuq.queue_crawl_finished.group_id IN (SELECT id FROM cleaned_groups)+ );+$$);