Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): rearchitect crawl kickoff (#2133)
Gergő Móricz65fee317a6c7
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex 3d6fd25b4..c030e10c4 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -68,7 +68,6 @@ jobs:PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgresTEST_SUITE_SELF_HOSTED: true- NUQ_REDUCE_NOISE: trueHOST: 0.0.0.0steps:- uses: actions/checkout@v5@@ -203,7 +202,6 @@ jobs:IDMUX_URL: ${{ secrets.IDMUX_URL }}LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres- NUQ_REDUCE_NOISE: trueHOST: 0.0.0.0steps:- uses: actions/checkout@v5diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.tsindex 41da3a5cb..22015f54d 100644--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts@@ -23,6 +23,13 @@ const logIgnoreList = ["No jobs to process","nuqHealthCheck metrics","nuqGetJobToProcess metrics",+ "Domain frequency processor",+ "billing operation to batch queue",+ "billing operation to queue",+ "billing operation for team",+ "Added billing operation to queue",+ "Index RF inserter found",+ "Redis connected",];if (process.env.TEST_SUITE_SELF_HOSTED) {diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.tsindex 4cae9a903..ceeacf509 100644--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts@@ -23,6 +23,13 @@ const logIgnoreList = ["No jobs to process","nuqHealthCheck metrics","nuqGetJobToProcess metrics",+ "Domain frequency processor",+ "billing operation to batch queue",+ "billing operation to queue",+ "billing operation for team",+ "Added billing operation to queue",+ "Index RF inserter found",+ "Redis connected",];if (process.env.TEST_SUITE_SELF_HOSTED) {diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 820bef824..60a66d703 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -4,7 +4,7 @@ import {checkTeamCredits,} from "../../services/billing/credit_billing";import { authenticateUser } from "../auth";-import { RateLimiterMode } from "../../types";+import { RateLimiterMode, ScrapeJobSingleUrls } from "../../types";import { logJob } from "../../services/logging/log_job";import { PageOptions, SearchOptions } from "../../lib/entities";import { search } from "../../search";@@ -23,6 +23,7 @@ import {import { fromV0Combo } from "../v2/types";import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";+import { defaultOrigin } from "../../lib/default-values";async function searchHelper(jobId: string,@@ -117,7 +118,8 @@ async function searchHelper(startTime: Date.now(),zeroDataRetention: false, // not supported on v0apiKeyId: api_key_id,- },+ origin: req.body.origin ?? defaultOrigin,+ } satisfies ScrapeJobSingleUrls,};});diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex a4b5cd2bb..f0899c616 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -41,6 +41,9 @@ export async function crawlErrorsController(res.status(200).json({errors: failedJobs.map(x => {+ if (x.data.mode !== "single_urls") {+ return null;+ }const error = deserializeTransportableError(x.failedReason!,) as TransportableError | null;diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex d856de2cf..07b079847 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -22,6 +22,7 @@ import { logger } from "../../lib/logger";import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getJobFromGCS } from "../../lib/gcs-jobs";import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";+import { ScrapeJobSingleUrls } from "../../types";configDotenv();export type PseudoJob<T> = {@@ -47,7 +48,7 @@ export type DBJob = {export async function getJob(id: string): Promise<PseudoJob<any> | null> {const [nuqJob, dbJob, gcsJob] = await Promise.all([- scrapeQueue.getJob(id),+ scrapeQueue.getJob(id) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,(process.env.USE_DB_AUTHENTICATION === "true"? supabaseGetJobById(id): null) as Promise<DBJob | null>,@@ -58,6 +59,10 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {if (!nuqJob && !dbJob) return null;+ if (nuqJob && nuqJob.data.mode !== "single_urls") {+ return null;+ }+const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;if (gcsJob === null && data) {logger.warn("GCS Job not found", {@@ -83,7 +88,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([- scrapeQueue.getJobs(ids),+ scrapeQueue.getJobs(ids) as Promise<NuQJob<ScrapeJobSingleUrls>[]>,process.env.USE_DB_AUTHENTICATION === "true"? supabaseGetJobsById(ids): [],diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex c04236470..dfbd8597a 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -134,8 +134,8 @@ export async function crawlController(await _addScrapeJobToBullMQ({- url: req.body.url,mode: "kickoff" as const,+ url: req.body.url,team_id: req.auth.team_id,crawlerOptions,scrapeOptions: sc.scrapeOptions,diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex baee96800..615a6537c 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -123,7 +123,7 @@ export async function scrapeController(});if (zeroDataRetention) {- await scrapeQueue.removeJob(jobId);+ await scrapeQueue.removeJob(jobId, logger);}if (e instanceof TransportableError) {@@ -143,7 +143,7 @@ export async function scrapeController(logger.info("Done with waitForJob");- await scrapeQueue.removeJob(jobId);+ await scrapeQueue.removeJob(jobId, logger);logger.info("Removed job from queue");diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 00b51ed26..b4d3dd6e0 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -12,7 +12,6 @@ import { billTeam } from "../../services/billing/credit_billing";import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";-import { Mode } from "../../types";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";@@ -119,7 +118,7 @@ async function scrapeSearchResult(await addScrapeJob({url: searchResult.url,- mode: "single_urls" as Mode,+ mode: "single_urls",team_id: options.teamId,scrapeOptions: {...scrapeOptions,@@ -157,7 +156,7 @@ async function scrapeSearchResult(teamId: options.teamId,origin: options.origin,});- await scrapeQueue.removeJob(jobId);+ await scrapeQueue.removeJob(jobId, logger);const document = {title: searchResult.title,diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 0e5aed0e0..10292e230 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -11,7 +11,6 @@ import {import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";-import { Mode } from "../../types";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";@@ -69,7 +68,7 @@ async function scrapeX402SearchResult(await addScrapeJob({url: searchResult.url,- mode: "single_urls" as Mode,+ mode: "single_urls",team_id: options.teamId,scrapeOptions: {...scrapeOptions,@@ -110,7 +109,7 @@ async function scrapeX402SearchResult(teamId: options.teamId,origin: options.origin,});- await scrapeQueue.removeJob(jobId);+ await scrapeQueue.removeJob(jobId, logger);const document = {title: searchResult.title,diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex a4b5cd2bb..f0899c616 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -41,6 +41,9 @@ export async function crawlErrorsController(res.status(200).json({errors: failedJobs.map(x => {+ if (x.data.mode !== "single_urls") {+ return null;+ }const error = deserializeTransportableError(x.failedReason!,) as TransportableError | null;diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.tsindex 6c4c6155b..91c1dd809 100644--- a/apps/api/src/controllers/v2/crawl-status-ws.ts+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts@@ -92,7 +92,10 @@ async function crawlStatusWS(])).map(x => x.id);- const newlyDoneJobs: PseudoJob<any>[] = await getJobs(newlyDoneJobIDs);+ const newlyDoneJobs: PseudoJob<any>[] = await getJobs(+ newlyDoneJobIDs,+ logger,+ );for (const job of newlyDoneJobs) {if (job.returnvalue) {@@ -145,7 +148,7 @@ async function crawlStatusWS(jobIDs = validJobIDs; // Use validJobIDs instead of jobIDs for further processing- const doneJobs = await getJobs(doneJobIDs);+ const doneJobs = await getJobs(doneJobIDs, logger);const data = doneJobs.map(x => x.returnvalue);await send(ws, {diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 3520e2063..098b9cbdc 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -22,6 +22,7 @@ import { logger } from "../../lib/logger";import { supabase_rr_service, supabase_service } from "../../services/supabase";import { getJobFromGCS } from "../../lib/gcs-jobs";import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";+import { ScrapeJobSingleUrls } from "../../types";configDotenv();export type PseudoJob<T> = {@@ -45,9 +46,15 @@ export type DBJob = {team_id: string;};-export async function getJob(id: string): Promise<PseudoJob<any> | null> {+export async function getJob(+ id: string,+ _logger = logger,+): Promise<PseudoJob<any> | null> {const [nuqJob, dbJob, gcsJob] = await Promise.all([- scrapeQueue.getJob(id),+ scrapeQueue.getJob(+ id,+ _logger,+ ) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,(process.env.USE_DB_AUTHENTICATION === "true"? supabaseGetJobById(id): null) as Promise<DBJob | null>,@@ -58,9 +65,13 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {if (!nuqJob && !dbJob) return null;+ if (nuqJob && nuqJob.data.mode !== "single_urls") {+ return null;+ }+const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;if (gcsJob === null && data) {- logger.warn("GCS Job not found", {+ _logger.warn("GCS Job not found", {jobId: id,});}@@ -81,9 +92,12 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {return job;}-export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {+export async function getJobs(+ ids: string[],+ _logger = logger,+): Promise<PseudoJob<any>[]> {const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([- scrapeQueue.getJobs(ids),+ scrapeQueue.getJobs(ids, _logger) as Promise<NuQJob<ScrapeJobSingleUrls>[]>,process.env.USE_DB_AUTHENTICATION === "true"? supabaseGetJobsById(ids): [],@@ -394,7 +408,7 @@ export async function crawlStatusController(for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) {const jobIds = doneJobs.slice(i * 50, (i + 1) * 50);- const jobs = await getJobs(jobIds);+ const jobs = await getJobs(jobIds, logger);for (const job of jobs) {if (job.status === "failed") {diff --git a/apps/api/src/controllers/v2/scrape-status.ts b/apps/api/src/controllers/v2/scrape-status.tsindex 7127169ab..a1064b4a7 100644--- a/apps/api/src/controllers/v2/scrape-status.ts+++ b/apps/api/src/controllers/v2/scrape-status.ts@@ -36,7 +36,7 @@ export async function scrapeStatusController(req: any, res: any) {});}- const jobData = await getJob(req.params.jobId);+ const jobData = await getJob(req.params.jobId, logger);const data = Array.isArray(jobData?.returnvalue)? jobData?.returnvalue[0]: jobData?.returnvalue;diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex 3663f0135..1eff281e4 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -120,7 +120,7 @@ export async function scrapeController(});if (zeroDataRetention) {- await scrapeQueue.removeJob(jobId);+ await scrapeQueue.removeJob(jobId, logger);}if (e instanceof TransportableError) {@@ -137,7 +137,7 @@ export async function scrapeController(}}- await scrapeQueue.removeJob(jobId);+ await scrapeQueue.removeJob(jobId, logger);if (!hasFormatOfType(req.body.formats, "rawHtml")) {if (doc && doc.rawHtml) {diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 8989abfa6..199ffae25 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -12,7 +12,6 @@ import { billTeam } from "../../services/billing/credit_billing";import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";-import { Mode } from "../../types";import { search } from "../../search/v2";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";@@ -22,7 +21,7 @@ import { getJobPriority } from "../../lib/job-priority";import { CostTracking } from "../../lib/cost-tracking";import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";import { supabase_service } from "../../services/supabase";-import { SearchResult, SearchV2Response } from "../../lib/entities";+import { SearchV2Response } from "../../lib/entities";import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";import { z } from "zod";@@ -78,7 +77,7 @@ async function startScrapeJob(await addScrapeJob({url: searchResult.url,- mode: "single_urls" as Mode,+ mode: "single_urls",team_id: options.teamId,scrapeOptions: {...options.scrapeOptions,@@ -140,7 +139,7 @@ async function scrapeSearchResult(origin: options.origin,});- await scrapeQueue.removeJob(jobId);+ await scrapeQueue.removeJob(jobId, logger);const document = {title: searchResult.title,diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex 3e6a1019a..a740f444a 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -103,10 +103,12 @@ const logger = {`${symbolColor}${symbol}${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${timing}${codeInfo}`,);},- processOutput(name: string, line: string) {+ processOutput(name: string, line: string, isReduceNoise: boolean) {const color = getProcessColor(name);const label = `${color}${name.padEnd(12)}${colors.reset}`;- console.log(`${label} ${line}`);+ if (!(line.includes("[nuq/metrics:") && isReduceNoise)) {+ console.log(`${label} ${line}`);+ }stream.write(`${name.padEnd(12)} ${line}\n`);},};@@ -152,6 +154,9 @@ function execForward(let displayCommand = "";const isWindows = process.platform === "win32";+ const isReduceNoise = env.NUQ_REDUCE_NOISE === "true";+ delete env.NUQ_REDUCE_NOISE;+if (typeof command === "string") {displayCommand = command;if (isWindows) {@@ -194,7 +199,7 @@ function execForward(const remainingBuffer = lines[lines.length - 1];completeLines.forEach(line => {- if (line.trim()) logger.processOutput(name, line);+ if (line.trim()) logger.processOutput(name, line, isReduceNoise);});if (isError) stderrBuffer = remainingBuffer;@@ -317,6 +322,9 @@ function startServices(command?: string[]): Services {process.argv[2] === "--start-docker"? "node dist/src/index.js": "pnpm server:production:nobuild",+ {+ NUQ_REDUCE_NOISE: "true",+ },);const worker = execForward(@@ -324,6 +332,9 @@ function startServices(command?: string[]): Services {process.argv[2] === "--start-docker"? "node dist/src/services/queue-worker.js": "pnpm worker:production",+ {+ NUQ_REDUCE_NOISE: "true",+ },);const nuqWorkers = Array.from({ length: 5 }, (_, i) =>@@ -346,6 +357,9 @@ function startServices(command?: string[]): Services {process.argv[2] === "--start-docker"? "node dist/src/services/indexing/index-worker.js": "pnpm index-worker:production",+ {+ NUQ_REDUCE_NOISE: "true",+ },): undefined;diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.tsindex 78ba0cda2..186835a68 100644--- a/apps/api/src/lib/crawl-redis.ts+++ b/apps/api/src/lib/crawl-redis.ts@@ -197,7 +197,7 @@ async function isCrawlFinished(id: string) {return ((await redisEvictConnection.scard("crawl:" + id + ":jobs_done")) ===(await redisEvictConnection.scard("crawl:" + id + ":jobs")) &&- (await redisEvictConnection.get("crawl:" + id + ":kickoff:finish")) !== null+ (await isCrawlKickoffFinished(id)));}@@ -207,7 +207,10 @@ export async function isCrawlKickoffFinished(id: string) {24 * 60 * 60,);return (- (await redisEvictConnection.get("crawl:" + id + ":kickoff:finish")) !== null+ (await redisEvictConnection.get("crawl:" + id + ":kickoff:finish")) !==+ null &&+ (await redisEvictConnection.scard("crawl:" + id + ":sitemap_jobs_done")) ===+ (await redisEvictConnection.scard("crawl:" + id + ":sitemap_jobs")));}diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex d7b70618e..e81582e0e 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -75,7 +75,7 @@ export async function scrapeDocument(const doc = await waitForJob(jobId, timeout, false, logger);try {- await scrapeQueue.removeJob(jobId);+ await scrapeQueue.removeJob(jobId, logger);} catch (error) {logger.warn("Error removing job from queue", { error, scrapeId: jobId });}diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex df7319b8e..a1acbc8ec 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -74,7 +74,7 @@ export async function scrapeDocument_F0(const doc = await waitForJob(jobId, timeout, false, logger);try {- await scrapeQueue.removeJob(jobId);+ await scrapeQueue.removeJob(jobId, logger);} catch (error) {logger.warn("Error removing job from queue", { error, scrapeId: jobId });}diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.tsindex 29d78e2d5..10a1e11a5 100644--- a/apps/api/src/lib/logger.ts+++ b/apps/api/src/lib/logger.ts@@ -35,17 +35,6 @@ const zeroDataRetentionFilter = winston.format(info => {return info;})();-const reduceNoiseFilter = winston.format(info => {- if (- (info.metadata?.module === "nuq/metrics" ||- info.module === "nuq/metrics") &&- process.env.NUQ_REDUCE_NOISE === "true"- ) {- return false; // Don't log this message- }- return info;-})();-export const logger = winston.createLogger({level: process.env.LOGGING_LEVEL?.toLowerCase() ?? "debug",format: winston.format.json({@@ -82,7 +71,6 @@ export const logger = winston.createLogger({: []),new winston.transports.Console({format: winston.format.combine(- reduceNoiseFilter,zeroDataRetentionFilter,winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),winston.format.metadata({diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.tsindex 0ef187f87..c80c7dd3b 100644--- a/apps/api/src/main/runWebScraper.ts+++ b/apps/api/src/main/runWebScraper.ts@@ -1,4 +1,4 @@-import { WebScraperOptions, RunWebScraperParams } from "../types";+import { ScrapeJobSingleUrls, RunWebScraperParams } from "../types";import { logger as _logger } from "../lib/logger";import { configDotenv } from "dotenv";import { scrapeURL, ScrapeUrlResponse } from "../scraper/scrapeURL";@@ -10,12 +10,11 @@ export async function startWebScraperPipeline({job,costTracking,}: {- job: NuQJob<WebScraperOptions>;+ job: NuQJob<ScrapeJobSingleUrls>;costTracking: CostTracking;}) {return await runWebScraper({url: job.data.url,- mode: job.data.mode,scrapeOptions: {...job.data.scrapeOptions,...(job.data.crawl_id@@ -34,7 +33,6 @@ export async function startWebScraperPipeline({team_id: job.data.team_id,bull_job_id: job.id,priority: job.priority,- is_scrape: job.data.is_scrape ?? false,is_crawl: !!(job.data.crawl_id && job.data.crawlerOptions !== null),urlInvisibleInCurrentCrawl:job.data.crawlerOptions?.urlInvisibleInCurrentCrawl ?? false,@@ -44,13 +42,11 @@ export async function startWebScraperPipeline({async function runWebScraper({url,- mode,scrapeOptions,internalOptions,team_id,bull_job_id,priority,- is_scrape = false,is_crawl = false,urlInvisibleInCurrentCrawl = false,costTracking,diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.tsnew file mode 100644index 000000000..fafbc872b--- /dev/null+++ b/apps/api/src/scraper/crawler/sitemap.ts@@ -0,0 +1,140 @@+import type { Logger } from "winston";+import { scrapeOptions, ScrapeOptions } from "../../controllers/v2/types";+import { logger as _logger } from "../../lib/logger";+import { Engine } from "../scrapeURL/engines";+import { scrapeURL } from "../scrapeURL";+import { CostTracking } from "../../lib/cost-tracking";+import { processSitemap } from "@mendable/firecrawl-rs";+import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";+import { gunzip } from "node:zlib";+import { promisify } from "node:util";++const useFireEngine =+ process.env.FIRE_ENGINE_BETA_URL !== "" &&+ process.env.FIRE_ENGINE_BETA_URL !== undefined;++type SitemapScrapeOptions = {+ url: string;+ maxAge: number;+ zeroDataRetention: boolean;+ location: ScrapeOptions["location"];+ crawlId: string;+ logger?: Logger;+};++type SitemapData = {+ urls: URL[];+ sitemaps: URL[];+};++const gunzipAsync = promisify(gunzip);++async function _getSitemapXMLGZ(+ options: SitemapScrapeOptions,+): Promise<string> {+ const { buffer } = await fetchFileToBuffer(options.url);+ const decompressed = await gunzipAsync(buffer);+ return decompressed.toString("utf-8");+}++async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {+ if (options.url.toLowerCase().endsWith(".gz")) {+ return await _getSitemapXMLGZ(options);+ }++ const isLocationSpecified =+ options.location && options.location.country !== "us-generic";++ const forceEngine: Engine[] = [+ ...(options.maxAge > 0 ? ["index" as const] : []),+ ...(isLocationSpecified && useFireEngine+ ? [+ "fire-engine;tlsclient" as const,+ "fire-engine;tlsclient;stealth" as const,+ ]+ : []),+ "fetch",+ ...(!isLocationSpecified && useFireEngine+ ? [+ "fire-engine;tlsclient" as const,+ "fire-engine;tlsclient;stealth" as const,+ ]+ : []),+ ];++ const response = await scrapeURL(+ "sitemap;" + options.crawlId,+ options.url,+ scrapeOptions.parse({+ formats: ["rawHtml"],+ maxAge: options.maxAge,+ ...(options.location ? { location: options.location } : {}),+ }),+ {+ forceEngine,+ v0DisableJsDom: true,+ // externalAbort: options.abort,+ teamId: "sitemap",+ zeroDataRetention: options.zeroDataRetention,+ crawlId: options.crawlId,+ },+ new CostTracking(),+ );++ if (+ response.success &&+ response.document.metadata.statusCode >= 200 &&+ response.document.metadata.statusCode < 300+ ) {+ return response.document.rawHtml!;+ } else if (!response.success) {+ throw new Error("Failed to scrape sitemap", { cause: response.error });+ } else {+ throw new Error("Failed to scrape sitemap", {+ cause: response.document.metadata.statusCode,+ });+ }+}++export async function scrapeSitemap(+ options: SitemapScrapeOptions,+): Promise<SitemapData> {+ const logger = (options.logger ?? _logger).child({+ module: "crawler",+ method: "scrapeSitemap",+ crawlId: options.crawlId,+ sitemapUrl: options.url,+ zeroDataRetention: options.zeroDataRetention,+ });++ logger.info("Scraping sitemap", {+ maxAge: options.maxAge,+ location: options.location,+ });++ const xml = await getSitemapXML(options);++ logger.info("Processing sitemap");++ const instructions = await processSitemap(xml);++ const sitemapData: SitemapData = {+ urls: [],+ sitemaps: [],+ };++ for (const instruction of instructions.instructions) {+ if (instruction.action === "recurse") {+ sitemapData.sitemaps.push(...instruction.urls.map(url => new URL(url)));+ } else if (instruction.action === "process") {+ sitemapData.urls.push(...instruction.urls.map(url => new URL(url)));+ }+ }++ logger.info("Processed sitemap", {+ urls: sitemapData.urls.length,+ sitemaps: sitemapData.sitemaps.length,+ });++ return sitemapData;+}diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.tsindex 358ae65bd..97b252629 100644--- a/apps/api/src/services/ab-test.ts+++ b/apps/api/src/services/ab-test.ts@@ -1,4 +1,4 @@-import { WebScraperOptions } from "../types";+import { ScrapeJobData } from "../types";import { logger as _logger } from "../lib/logger";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import {@@ -8,7 +8,7 @@ import {FireEngineScrapeRequestTLSClient,} from "../scraper/scrapeURL/engines/fire-engine/scrape";-export function abTestJob(webScraperOptions: WebScraperOptions) {+export function abTestJob(webScraperOptions: ScrapeJobData) {// Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATEconst abLogger = _logger.child({ method: "ABTestToStaging" });try {diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 67141224a..da0bda268 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,5 +1,5 @@import { v4 as uuidv4 } from "uuid";-import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";+import { NotificationType, RateLimiterMode, ScrapeJobData } from "../types";import {cleanOldConcurrencyLimitEntries,getConcurrencyLimitActiveJobs,@@ -55,11 +55,13 @@ async function _addScrapeJobToConcurrencyQueue(}export async function _addScrapeJobToBullMQ(- webScraperOptions: WebScraperOptions,+ webScraperOptions: ScrapeJobData,jobId: string,priority: number = 0,-): Promise<NuQJob<WebScraperOptions>> {- abTestJob(webScraperOptions);+): Promise<NuQJob<ScrapeJobData>> {+ if (webScraperOptions.mode === "single_urls") {+ abTestJob(webScraperOptions);+ }if (webScraperOptions && webScraperOptions.team_id) {await pushConcurrencyLimitActiveJob(@@ -70,7 +72,7 @@ export async function _addScrapeJobToBullMQ(if (webScraperOptions.crawl_id) {const sc = await getCrawl(webScraperOptions.crawl_id);- if (webScraperOptions.crawlerOptions?.delay || sc?.maxConcurrency) {+ if (sc?.crawlerOptions?.delay || sc?.maxConcurrency) {await pushCrawlConcurrencyLimitActiveJob(webScraperOptions.crawl_id,jobId,@@ -84,11 +86,11 @@ export async function _addScrapeJobToBullMQ(}async function addScrapeJobRaw(- webScraperOptions: WebScraperOptions,+ webScraperOptions: ScrapeJobData,jobId: string,priority: number = 0,directToBullMQ: boolean = false,-): Promise<NuQJob<WebScraperOptions> | null> {+): Promise<NuQJob<ScrapeJobData> | null> {let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;let currentActiveConcurrency = 0;let maxConcurrency = 0;@@ -124,7 +126,8 @@ async function addScrapeJobRaw(webScraperOptions.team_id,false,true,- webScraperOptions.is_extract+ webScraperOptions.mode === "single_urls" &&+ webScraperOptions.from_extract? RateLimiterMode.Extract: RateLimiterMode.Crawl,)@@ -181,11 +184,11 @@ async function addScrapeJobRaw(}export async function addScrapeJob(- webScraperOptions: WebScraperOptions,+ webScraperOptions: ScrapeJobData,jobId: string = uuidv4(),priority: number = 0,directToBullMQ: boolean = false,-): Promise<NuQJob<WebScraperOptions> | null> {+): Promise<NuQJob<ScrapeJobData> | null> {return await addScrapeJobRaw(webScraperOptions,jobId,@@ -197,7 +200,7 @@ export async function addScrapeJob(export async function addScrapeJobs(jobs: {jobId: string;- data: WebScraperOptions;+ data: ScrapeJobData;priority: number;}[],) {@@ -207,7 +210,7 @@ export async function addScrapeJobs(string,{jobId: string;- data: WebScraperOptions;+ data: ScrapeJobData;priority: number;}[]>();@@ -222,13 +225,13 @@ export async function addScrapeJobs(for (const [teamId, teamJobs] of jobsByTeam) {// == Buckets for jobs ==let jobsForcedToCQ: {- data: WebScraperOptions;+ data: ScrapeJobData;jobId: string;priority: number;}[] = [];let jobsPotentiallyInCQ: {- data: WebScraperOptions;+ data: ScrapeJobData;jobId: string;priority: number;}[] = [];@@ -237,14 +240,14 @@ export async function addScrapeJobs(const jobsByCrawlID = new Map<string,{- data: WebScraperOptions;+ data: ScrapeJobData;jobId: string;priority: number;}[]>();const jobsWithoutCrawlID: {- data: WebScraperOptions;+ data: ScrapeJobData;jobId: string;priority: number;}[] = [];@@ -297,7 +300,7 @@ export async function addScrapeJobs(teamId,false,true,- jobs[0].data.from_extract+ jobs[0].data.mode === "single_urls" && jobs[0].data.from_extract? RateLimiterMode.Extract: RateLimiterMode.Crawl,)@@ -362,7 +365,7 @@ export async function addScrapeJobs(}export async function waitForJob(- job: NuQJob<WebScraperOptions> | string,+ job: NuQJob<ScrapeJobData> | string,timeout: number | null,zeroDataRetention: boolean,logger: Logger = _logger,@@ -374,7 +377,11 @@ export async function waitForJob(try {doc = await Promise.race([- scrapeQueue.waitForJob(jobId, timeout !== null ? timeout + 100 : null),+ scrapeQueue.waitForJob(+ jobId,+ timeout !== null ? timeout + 100 : null,+ logger,+ ),timeout !== null? new Promise<Document>((_resolve, reject) => {setTimeout(() => {diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex 7d4eabab0..af3489d44 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -21,37 +21,47 @@ import { logJob } from "../logging/log_job";import { createWebhookSender, WebhookEvent } from "../webhook";import { hasFormatOfType } from "../../lib/format-utils";import type { NuQJob } from "./nuq";+import { ScrapeJobData } from "../../types";++export async function finishCrawlIfNeeded(+ job: NuQJob<ScrapeJobData>,+ sc: StoredCrawl,+) {+ const crawlId = job.data.crawl_id!;++ if (!crawlId) {+ return;+ }-export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {const logger = _logger.child({module: "queue-worker",method: "finishCrawlIfNeeded",jobId: job.id,scrapeId: job.id,- crawlId: job.data.crawl_id,+ crawlId,zeroDataRetention: sc.internalOptions.zeroDataRetention,});- if (await finishCrawlPre(job.data.crawl_id, logger)) {+ if (await finishCrawlPre(crawlId, logger)) {logger.info("Crawl is pre-finished, checking if we need to add more jobs");if (- job.data.crawlerOptions &&+ sc.crawlerOptions &&!(await redisEvictConnection.exists(- "crawl:" + job.data.crawl_id + ":invisible_urls",+ "crawl:" + crawlId + ":invisible_urls",))) {await redisEvictConnection.set(- "crawl:" + job.data.crawl_id + ":invisible_urls",+ "crawl:" + crawlId + ":invisible_urls","done","EX",60 * 60 * 24,);- const sc = (await getCrawl(job.data.crawl_id))!;+ const sc = (await getCrawl(crawlId))!;const visitedUrls = new Set(await redisEvictConnection.smembers(- "crawl:" + job.data.crawl_id + ":visited_unique",+ "crawl:" + crawlId + ":visited_unique",),);@@ -82,11 +92,11 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {});const crawler = crawlToCrawler(- job.data.crawl_id,+ crawlId,sc,- (await getACUCTeam(job.data.team_id))?.flags ?? null,+ (await getACUCTeam(job.data.team_id!))?.flags ?? null,sc.originUrl!,- job.data.crawlerOptions,+ sc.crawlerOptions,);const univistedUrls = await crawler.filterLinks(@@ -98,8 +108,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {const addableJobCount =sc.crawlerOptions.limit === undefined? Infinity- : sc.crawlerOptions.limit -- (await getDoneJobsOrderedLength(job.data.crawl_id));+ : sc.crawlerOptions.limit - (await getDoneJobsOrderedLength(crawlId));if (univistedUrls.links.length !== 0 && addableJobCount > 0) {logger.info("Adding jobs", {@@ -116,14 +125,14 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {mode: "single_urls" as const,team_id: job.data.team_id,crawlerOptions: {- ...job.data.crawlerOptions,+ ...sc.crawlerOptions,urlInvisibleInCurrentCrawl: true,},- scrapeOptions: job.data.scrapeOptions,+ scrapeOptions: sc.scrapeOptions,internalOptions: sc.internalOptions,origin: job.data.origin,integration: job.data.integration,- crawl_id: job.data.crawl_id,+ crawl_id: crawlId,sitemapped: true,webhook: job.data.webhook,v1: job.data.v1,@@ -135,7 +144,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {});const lockedIds = await lockURLsIndividually(- job.data.crawl_id,+ crawlId,sc,jobs.map(x => ({ id: x.jobId, url: x.data.url })),);@@ -143,7 +152,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {lockedIds.find(y => y.id === x.jobId),);await addCrawlJobs(- job.data.crawl_id,+ crawlId,lockedJobs.map(x => x.jobId),logger,);@@ -154,7 +163,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {lockedJobs: lockedJobs.length,});- await unPreFinishCrawl(job.data.crawl_id);+ await unPreFinishCrawl(crawlId);return;} else {logger.info(@@ -165,10 +174,10 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {}logger.info("Finishing crawl");- await finishCrawl(job.data.crawl_id, logger);+ await finishCrawl(crawlId, logger);if (!job.data.v1) {- const jobIDs = await getCrawlJobs(job.data.crawl_id);+ const jobIDs = await getCrawlJobs(crawlId);const jobs = (await getJobs(jobIDs)).sort((a, b) => a.timestamp - b.timestamp,@@ -190,44 +199,30 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {await logJob({- job_id: job.data.crawl_id,+ job_id: crawlId,success: jobStatus === "completed",message: sc.cancelled ? "Cancelled" : undefined,num_docs: fullDocs.length,docs: [],time_taken: (Date.now() - sc.createdAt) / 1000,team_id: job.data.team_id,- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",+ mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",url: sc.originUrl!,scrapeOptions: sc.scrapeOptions,crawlerOptions: sc.crawlerOptions,- origin: job.data.origin,+ origin: sc.originUrl!,integration: job.data.integration,zeroDataRetention: job.data.zeroDataRetention,},false,- job.data.internalOptions?.bypassBilling ?? false,+ sc.internalOptions?.bypassBilling ?? false,);- const data = {- success: jobStatus !== "failed",- result: {- links: fullDocs.map(doc => {- return {- content: doc,- source: doc?.metadata?.sourceURL ?? doc?.url ?? "",- };- }),- },- project_id: job.data.project_id,- docs: fullDocs,- };-// v0 web hooks, call when done with all the dataif (!job.data.v1) {const sender = await createWebhookSender({teamId: job.data.team_id,- jobId: job.data.crawl_id,+ jobId: crawlId,webhook: job.data.webhook,v0: true,});@@ -240,7 +235,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {},source: doc?.metadata?.sourceURL ?? doc?.url ?? "",}));- if (job.data.crawlerOptions !== null) {+ if (sc.crawlerOptions !== null) {sender.send(WebhookEvent.CRAWL_COMPLETED, {success: true,data: documents,@@ -254,7 +249,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {}}} else {- const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);+ const num_docs = await getDoneJobsOrderedLength(crawlId);const jobStatus = sc.cancelled ? "failed" : "completed";let credits_billed = null;@@ -263,7 +258,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {const creditsRpc = await supabase_service.rpc("credits_billed_by_crawl_id_1",{- i_crawl_id: job.data.crawl_id,+ i_crawl_id: crawlId,},);@@ -278,18 +273,18 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {await logJob({- job_id: job.data.crawl_id,+ job_id: crawlId,success: jobStatus === "completed",message: sc.cancelled ? "Cancelled" : undefined,num_docs,docs: [],time_taken: (Date.now() - sc.createdAt) / 1000,- team_id: job.data.team_id,+ team_id: sc.team_id,scrapeOptions: sc.scrapeOptions,- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",+ mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",url:sc?.originUrl ??- (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),+ (sc.crawlerOptions === null ? "Batch Scrape" : "Unknown"),crawlerOptions: sc.crawlerOptions,origin: job.data.origin,integration: job.data.integration,@@ -297,19 +292,19 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {zeroDataRetention: job.data.zeroDataRetention,},true,- job.data.internalOptions?.bypassBilling ?? false,+ sc.internalOptions?.bypassBilling ?? false,);// v1 web hooks, call when done with no data, but with event completedif (job.data.v1 && job.data.webhook) {const sender = await createWebhookSender({teamId: job.data.team_id,- jobId: job.data.crawl_id,+ jobId: crawlId,webhook: job.data.webhook,v0: false,});if (sender) {- if (job.data.crawlerOptions !== null) {+ if (sc.crawlerOptions !== null) {sender.send(WebhookEvent.CRAWL_COMPLETED, {success: true,data: [],diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 993603a7a..d7dedde3b 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -1,6 +1,7 @@import { Logger } from "winston";import { logger } from "../../lib/logger";import { Client, Pool } from "pg";+import { type ScrapeJobData } from "../../types";// === Basics@@ -142,6 +143,7 @@ class NuQ<JobData = any, JobReturnValue = any> {public async getJob(id: string,+ _logger = logger,): Promise<NuQJob<JobData, JobReturnValue> | null> {const start = Date.now();try {@@ -154,7 +156,7 @@ class NuQ<JobData = any, JobReturnValue = any> {).rows[0],);} finally {- logger.info("nuqGetJob metrics", {+ _logger.info("nuqGetJob metrics", {module: "nuq/metrics",method: "nuqGetJob",duration: Date.now() - start,@@ -165,6 +167,7 @@ class NuQ<JobData = any, JobReturnValue = any> {public async getJobs(ids: string[],+ _logger: Logger = logger,): Promise<NuQJob<JobData, JobReturnValue>[]> {if (ids.length === 0) return [];@@ -177,7 +180,7 @@ class NuQ<JobData = any, JobReturnValue = any> {)).rows.map(row => this.rowToJob(row)!);} finally {- logger.info("nuqGetJobs metrics", {+ _logger.info("nuqGetJobs metrics", {module: "nuq/metrics",method: "nuqGetJobs",duration: Date.now() - start,@@ -215,6 +218,7 @@ class NuQ<JobData = any, JobReturnValue = any> {public async getJobsWithStatuses(ids: string[],statuses: NuQJobStatus[],+ _logger: Logger = logger,): Promise<NuQJob<JobData, JobReturnValue>[]> {if (ids.length === 0) return [];@@ -227,7 +231,7 @@ class NuQ<JobData = any, JobReturnValue = any> {)).rows.map(row => this.rowToJob(row)!);} finally {- logger.info("nuqGetJobsWithStatuses metrics", {+ _logger.info("nuqGetJobsWithStatuses metrics", {module: "nuq/metrics",method: "nuqGetJobsWithStatuses",duration: Date.now() - start,@@ -237,7 +241,10 @@ class NuQ<JobData = any, JobReturnValue = any> {}}- public async removeJob(id: string): Promise<boolean> {+ public async removeJob(+ id: string,+ _logger: Logger = logger,+ ): Promise<boolean> {const start = Date.now();try {return (@@ -248,7 +255,7 @@ class NuQ<JobData = any, JobReturnValue = any> {).rowCount !== 0);} finally {- logger.info("nuqRemoveJob metrics", {+ _logger.info("nuqRemoveJob metrics", {module: "nuq/metrics",method: "nuqRemoveJob",duration: Date.now() - start,@@ -257,7 +264,10 @@ class NuQ<JobData = any, JobReturnValue = any> {}}- public async removeJobs(ids: string[]): Promise<number> {+ public async removeJobs(+ ids: string[],+ _logger: Logger = logger,+ ): Promise<number> {if (ids.length === 0) return 0;const start = Date.now();@@ -271,7 +281,7 @@ class NuQ<JobData = any, JobReturnValue = any> {).rowCount ?? 0);} finally {- logger.info("nuqRemoveJobs metrics", {+ _logger.info("nuqRemoveJobs metrics", {module: "nuq/metrics",method: "nuqRemoveJobs",duration: Date.now() - start,@@ -315,6 +325,7 @@ class NuQ<JobData = any, JobReturnValue = any> {public waitForJob(id: string,timeout: number | null,+ _logger: Logger = logger,): Promise<JobReturnValue> {const done = new Promise<JobReturnValue>((async (resolve, reject) => {@@ -332,7 +343,7 @@ class NuQ<JobData = any, JobReturnValue = any> {const listener = async function (_msg: "completed" | "failed") {if (timer) clearTimeout(timer);- const job = await this.getJob(id);+ const job = await this.getJob(id, _logger);if (!job) {reject(new Error("Job raced out while waiting for it"));} else {@@ -351,7 +362,7 @@ class NuQ<JobData = any, JobReturnValue = any> {}try {- const job = await this.getJob(id);+ const job = await this.getJob(id, _logger);if (job && ["completed", "failed"].includes(job.status)) {this.removeListener(id, listener);if (timer) clearTimeout(timer);@@ -363,7 +374,7 @@ class NuQ<JobData = any, JobReturnValue = any> {return;}} catch (e) {- logger.warn("nuqGetJob ensure check failed", {+ _logger.warn("nuqGetJob ensure check failed", {module: "nuq",method: "nuqWaitForJob",error: e,@@ -374,7 +385,7 @@ class NuQ<JobData = any, JobReturnValue = any> {const timeoutAt = timeout !== null ? Date.now() + timeout : null;const poll = async function poll() {try {- const job = await this.getJob(id);+ const job = await this.getJob(id, _logger);if (job && ["completed", "failed"].includes(job.status)) {if (job.status === "completed") {return resolve(job.returnvalue!);@@ -558,7 +569,7 @@ export async function nuqHealthCheck(): Promise<boolean> {// === Instances-export const scrapeQueue = new NuQ("nuq.queue_scrape");+export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape");// === Cleanupdiff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex e40f44c69..72dff9795 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -25,7 +25,12 @@ import {saveCrawl,StoredCrawl,} from "../../lib/crawl-redis";-import { addScrapeJob, addScrapeJobs } from "../queue-jobs";+import {+ _addScrapeJobToBullMQ,+ addScrapeJob,+ addScrapeJobs,+} from "../queue-jobs";+import psl from "psl";import { getJobPriority } from "../../lib/job-priority";import { Document, scrapeOptions, TeamFlags } from "../../controllers/v2/types";import { hasFormatOfType } from "../../lib/format-utils";@@ -60,6 +65,13 @@ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";import { resourceFromAttributes } from "@opentelemetry/resources";import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";import type { NuQJob } from "./nuq";+import {+ ScrapeJobData,+ ScrapeJobKickoff,+ ScrapeJobKickoffSitemap,+ ScrapeJobSingleUrls,+} from "../../types";+import { scrapeSitemap } from "../../scraper/crawler/sitemap";const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));@@ -138,7 +150,7 @@ async function billScrapeJob(return creditsToBeBilled;}-async function processJob(job: NuQJob<any>) {+async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {const logger = _logger.child({module: "queue-worker",method: "processJob",@@ -224,7 +236,6 @@ async function processJob(job: NuQJob<any>) {},],},- project_id: job.data.project_id,document: doc,};@@ -441,7 +452,7 @@ async function processJob(job: NuQJob<any>) {job.data.internalOptions?.bypassBilling ?? false,);- if (job.data.mode !== "crawl" && job.data.v1) {+ if (job.data.v1) {const sender = await createWebhookSender({teamId: job.data.team_id,jobId: job.data.crawl_id,@@ -574,7 +585,6 @@ async function processJob(job: NuQJob<any>) {const data = {success: false,document: null,- project_id: job.data.project_id,error:error instanceof Error? error@@ -583,7 +593,7 @@ async function processJob(job: NuQJob<any>) {: new Error(JSON.stringify(error)),};- if (job.data.mode === "crawl" || job.data.crawl_id) {+ if (job.data.crawl_id) {const sender = await createWebhookSender({teamId: job.data.team_id,jobId: (job.data.crawl_id ?? job.id) as string,@@ -696,7 +706,55 @@ async function kickoffGetIndexLinks(return validIndexLinks;}-async function processKickoffJob(job: NuQJob<any>) {+async function addKickoffSitemapJob(+ sitemapUrl: string,+ sourceJob: NuQJob<ScrapeJobKickoff | ScrapeJobKickoffSitemap>,+ sc: StoredCrawl,+ logger: Logger,+) {+ const sitemapLocked =+ (await redisEvictConnection.sadd(+ "crawl:" + sourceJob.data.crawl_id + ":sitemaps",+ sitemapUrl,+ )) === 1;+ await redisEvictConnection.expire(+ "crawl:" + sourceJob.data.crawl_id + ":sitemaps",+ 24 * 60 * 60,+ );+ if (!sitemapLocked) {+ logger.debug("Sitemap already hit, skipping...", { sitemap: sitemapUrl });+ return;+ }++ const jobId = uuidv4();+ await _addScrapeJobToBullMQ(+ {+ mode: "kickoff_sitemap" as const,+ team_id: sourceJob.data.team_id,+ zeroDataRetention:+ sourceJob.data.zeroDataRetention || (sc.zeroDataRetention ?? false),+ sitemapUrl: sitemapUrl,+ origin: sourceJob.data.origin,+ integration: sourceJob.data.integration,+ crawl_id: sourceJob.data.crawl_id,+ webhook: sourceJob.data.webhook,+ v1: sourceJob.data.v1,+ apiKeyId: sourceJob.data.apiKeyId,+ } satisfies ScrapeJobKickoffSitemap,+ jobId,+ 21,+ );+ await redisEvictConnection.sadd(+ "crawl:" + sourceJob.data.crawl_id + ":sitemap_jobs",+ jobId,+ );+ await redisEvictConnection.expire(+ "crawl:" + sourceJob.data.crawl_id + ":sitemap_jobs",+ 24 * 60 * 60,+ );+}++async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {const logger = _logger.child({module: "queue-worker",method: "processKickoffJob",@@ -757,68 +815,36 @@ async function processKickoffJob(job: NuQJob<any>) {}}- const sitemap = sc.crawlerOptions.ignoreSitemap- ? 0- : await crawler.tryGetSitemap(async urls => {- if (urls.length === 0) return;-- logger.debug("Using sitemap chunk of length " + urls.length, {- sitemapLength: urls.length,- });-- let jobPriority = await getJobPriority({- team_id: job.data.team_id,- basePriority: 21,- });- logger.debug("Using job priority " + jobPriority, { jobPriority });-- const jobs = urls.map(url => {- const uuid = uuidv4();- return {- data: {- url,- mode: "single_urls" as const,- team_id: job.data.team_id,- crawlerOptions: job.data.crawlerOptions,- scrapeOptions: job.data.scrapeOptions,- internalOptions: sc.internalOptions,- origin: job.data.origin,- integration: job.data.integration,- crawl_id: job.data.crawl_id,- sitemapped: true,- webhook: job.data.webhook,- v1: job.data.v1,- zeroDataRetention: job.data.zeroDataRetention,- apiKeyId: job.data.apiKeyId,- },- jobId: uuid,- priority: jobPriority,- };- });-- logger.debug("Locking URLs...");- const lockedIds = await lockURLsIndividually(- job.data.crawl_id,- sc,- jobs.map(x => ({ id: x.jobId, url: x.data.url })),- );- const lockedJobs = jobs.filter(x =>- lockedIds.find(y => y.id === x.jobId),- );- logger.debug("Adding scrape jobs to Redis...");- await addCrawlJobs(- job.data.crawl_id,- lockedJobs.map(x => x.jobId),- logger,- );- logger.debug("Adding scrape jobs to BullMQ...");- await addScrapeJobs(lockedJobs);- });-- if (sitemap === 0) {- logger.debug("Sitemap not found or ignored.", {- ignoreSitemap: sc.crawlerOptions.ignoreSitemap,- });+ if (!sc.crawlerOptions.ignoreSitemap) {+ if (job.data.url.endsWith(".xml") || job.data.url.endsWith(".xml.gz")) {+ await addKickoffSitemapJob(job.data.url, job, sc, logger);+ } else {+ const urlObj = new URL(job.data.url);++ const attempts: string[] = crawler.robots.getSitemaps();++ // Append sitemap.xml+ const urlWithSitemap = new URL(urlObj.href);+ urlWithSitemap.pathname =+ urlWithSitemap.pathname ++ (urlObj.pathname.endsWith("/") ? "" : "/") ++ "sitemap.xml";+ urlWithSitemap.search = "";+ urlWithSitemap.hash = "";+ attempts.push(urlWithSitemap.href);++ // Base sitemap.xml+ attempts.push(new URL("/sitemap.xml", urlObj.href).href);++ // Root domain sitemap.xml+ const urlRootSitemap = new URL("/sitemap.xml", urlObj.href);+ urlRootSitemap.hostname = psl.parse(urlObj.hostname).domain;+ attempts.push(urlRootSitemap.href);++ for (const attempt of attempts) {+ await addKickoffSitemapJob(attempt, job, sc, logger);+ }+ }}const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);@@ -894,7 +920,130 @@ async function processKickoffJob(job: NuQJob<any>) {}}-export const processJobInternal = async (job: NuQJob<any>) => {+async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {+ const logger = _logger.child({+ module: "queue-worker",+ method: "processKickoffSitemapJob",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data.crawl_id,+ zeroDataRetention: job.data.zeroDataRetention ?? false,+ });++ const sc = await getCrawl(job.data.crawl_id);++ try {+ if (!sc) {+ logger.error("Crawl not found");+ return { success: false, error: "Crawl not found" };+ }++ const crawler = crawlToCrawler(+ job.data.crawl_id,+ sc,+ (await getACUCTeam(job.data.team_id))?.flags ?? null,+ );++ const results = await scrapeSitemap({+ url: job.data.sitemapUrl,+ maxAge: 48 * 60 * 60 * 1000,+ zeroDataRetention: job.data.zeroDataRetention ?? false,+ location: job.data.location,+ crawlId: job.data.crawl_id,+ logger,+ });++ const passingURLs = (+ await crawler.filterLinks(+ [...new Set(results.urls.map(x => x.href))].filter(+ url => crawler.filterURL(url, sc.originUrl!).allowed,+ ),+ Infinity,+ sc.crawlerOptions.maxDepth ?? 10,+ false,+ )+ ).links;++ if (passingURLs.length > 0) {+ logger.debug("Using urls of length " + passingURLs.length, {+ urlsLength: passingURLs.length,+ });++ const jobPriority = await getJobPriority({+ team_id: job.data.team_id,+ basePriority: 21,+ });++ const jobs = passingURLs.map(url => ({+ data: {+ url: url,+ mode: "single_urls" as const,+ team_id: job.data.team_id,+ crawlerOptions: sc.crawlerOptions,+ scrapeOptions: sc.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ integration: job.data.integration,+ crawl_id: job.data.crawl_id,+ sitemapped: true,+ webhook: job.data.webhook,+ v1: job.data.v1,+ zeroDataRetention:+ job.data.zeroDataRetention || (sc.zeroDataRetention ?? false),+ apiKeyId: job.data.apiKeyId,+ } satisfies ScrapeJobSingleUrls,+ jobId: uuidv4(),+ priority: jobPriority,+ }));++ const urls = await lockURLsIndividually(+ job.data.crawl_id,+ sc,+ jobs.map(x => ({ id: x.jobId, url: x.data.url })),+ );+ const winningIds = new Set(urls.map(x => x.id));+ await addCrawlJobs(+ job.data.crawl_id,+ urls.map(x => x.id),+ logger,+ );+ await addScrapeJobs(jobs.filter(x => winningIds.has(x.jobId)));++ logger.debug("Done queueing jobs!");+ }++ if (results.sitemaps.length > 0) {+ logger.debug("Using sitemaps of length " + results.sitemaps.length, {+ sitemapsLength: results.sitemaps.length,+ });++ for (const sitemap of results.sitemaps) {+ await addKickoffSitemapJob(sitemap.href, job, sc, logger);+ }++ logger.debug("Done queueing sitemap jobs!");+ }+ return { success: true };+ } catch (error) {+ logger.error("An error occurred!", { error });+ return { success: false, error };+ } finally {+ await redisEvictConnection.sadd(+ "crawl:" + job.data.crawl_id + ":sitemap_jobs_done",+ job.id,+ );+ await redisEvictConnection.expire(+ "crawl:" + job.data.crawl_id + ":sitemap_jobs_done",+ 24 * 60 * 60,+ );++ if (sc) {+ await finishCrawlIfNeeded(job, sc);+ }+ }+}++export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {const logger = _logger.child({module: "queue-worker",method: "processJobInternal",@@ -919,15 +1068,26 @@ export const processJobInternal = async (job: NuQJob<any>) => {await addJobPriority(job.data.team_id, job.id);try {- if (job.data?.mode === "kickoff") {- const result = await processKickoffJob(job);+ if (job.data.mode === "kickoff") {+ const result = await processKickoffJob(+ job as NuQJob<ScrapeJobKickoff>,+ );+ if (result.success) {+ return null;+ } else {+ throw (result as any).error;+ }+ } else if (job.data.mode === "kickoff_sitemap") {+ const result = await processKickoffSitemapJob(+ job as NuQJob<ScrapeJobKickoffSitemap>,+ );if (result.success) {return null;} else {throw (result as any).error;}} else {- const result = await processJob(job);+ const result = await processJob(job as NuQJob<ScrapeJobSingleUrls>);if (result.success) {try {if (job.data.team_id) {diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex 28aced098..0690ec1e2 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -11,18 +11,27 @@ import { InternalOptions } from "./scraper/scrapeURL";import type { CostTracking } from "./lib/cost-tracking";import { webhookSchema } from "./services/webhook/schema";-type Mode = "crawl" | "single_urls" | "sitemap" | "kickoff";+type ScrapeJobCommon = {+ concurrencyLimited?: boolean;+ team_id: string;+ zeroDataRetention: boolean;+};++export type ScrapeJobData = ScrapeJobCommon &+ (+ | ScrapeJobSingleUrlsUnique+ | ScrapeJobKickoffUnique+ | ScrapeJobKickoffSitemapUnique+ );-export { Mode };+type ScrapeJobSingleUrlsUnique = {+ mode: "single_urls";-export interface WebScraperOptions {url: string;- mode: Mode;crawlerOptions?: any;scrapeOptions: BaseScrapeOptions;internalOptions?: InternalOptions;- team_id: string;- origin?: string;+ origin: string;crawl_id?: string;sitemapped?: boolean;webhook?: z.infer<typeof webhookSchema>;@@ -38,22 +47,53 @@ export interface WebScraperOptions {from_extract?: boolean;startTime?: number;- zeroDataRetention: boolean;sentry?: any;is_extract?: boolean;- concurrencyLimited?: boolean;apiKeyId: number | null;-}+};++export type ScrapeJobSingleUrls = ScrapeJobCommon & ScrapeJobSingleUrlsUnique;++type ScrapeJobKickoffUnique = {+ mode: "kickoff";++ url: string;+ crawlerOptions?: any;+ scrapeOptions: BaseScrapeOptions;+ internalOptions?: InternalOptions;+ origin: string;+ integration?: string | null;+ crawl_id: string;+ webhook?: z.infer<typeof webhookSchema>;+ v1: boolean;+ apiKeyId: number | null;+};++export type ScrapeJobKickoff = ScrapeJobCommon & ScrapeJobKickoffUnique;++type ScrapeJobKickoffSitemapUnique = {+ mode: "kickoff_sitemap";++ crawl_id: string;+ sitemapUrl: string;+ location?: ScrapeOptions["location"];+ origin: string;+ integration?: string | null;+ webhook?: z.infer<typeof webhookSchema>;+ v1: boolean;+ apiKeyId: number | null;+};++export type ScrapeJobKickoffSitemap = ScrapeJobCommon &+ ScrapeJobKickoffSitemapUnique;export interface RunWebScraperParams {url: string;- mode: Mode;scrapeOptions: ScrapeOptions;internalOptions?: InternalOptions;team_id: string;bull_job_id: string;priority?: number;- is_scrape?: boolean;is_crawl?: boolean;urlInvisibleInCurrentCrawl?: boolean;costTracking: CostTracking;