Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): rearchitect crawl kickoff (#2133)

Gergő Móricz65fee317a6c7
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index 3d6fd25b4..c030e10c4 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -68,7 +68,6 @@ jobs:
PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}
NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
TEST_SUITE_SELF_HOSTED: true
- NUQ_REDUCE_NOISE: true
HOST: 0.0.0.0
steps:
- uses: actions/checkout@v5
@@ -203,7 +202,6 @@ jobs:
IDMUX_URL: ${{ secrets.IDMUX_URL }}
LOG_ENCRYPTION_KEY: ${{ secrets.LOG_ENCRYPTION_KEY }}
NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
- NUQ_REDUCE_NOISE: true
HOST: 0.0.0.0
steps:
- uses: actions/checkout@v5
diff --git a/apps/api/src/__tests__/snips/v1/zdr.test.ts b/apps/api/src/__tests__/snips/v1/zdr.test.ts
index 41da3a5cb..22015f54d 100644
--- a/apps/api/src/__tests__/snips/v1/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v1/zdr.test.ts
@@ -23,6 +23,13 @@ const logIgnoreList = [
"No jobs to process",
"nuqHealthCheck metrics",
"nuqGetJobToProcess metrics",
+ "Domain frequency processor",
+ "billing operation to batch queue",
+ "billing operation to queue",
+ "billing operation for team",
+ "Added billing operation to queue",
+ "Index RF inserter found",
+ "Redis connected",
];
if (process.env.TEST_SUITE_SELF_HOSTED) {
diff --git a/apps/api/src/__tests__/snips/v2/zdr.test.ts b/apps/api/src/__tests__/snips/v2/zdr.test.ts
index 4cae9a903..ceeacf509 100644
--- a/apps/api/src/__tests__/snips/v2/zdr.test.ts
+++ b/apps/api/src/__tests__/snips/v2/zdr.test.ts
@@ -23,6 +23,13 @@ const logIgnoreList = [
"No jobs to process",
"nuqHealthCheck metrics",
"nuqGetJobToProcess metrics",
+ "Domain frequency processor",
+ "billing operation to batch queue",
+ "billing operation to queue",
+ "billing operation for team",
+ "Added billing operation to queue",
+ "Index RF inserter found",
+ "Redis connected",
];
if (process.env.TEST_SUITE_SELF_HOSTED) {
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 820bef824..60a66d703 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -4,7 +4,7 @@ import {
checkTeamCredits,
} from "../../services/billing/credit_billing";
import { authenticateUser } from "../auth";
-import { RateLimiterMode } from "../../types";
+import { RateLimiterMode, ScrapeJobSingleUrls } from "../../types";
import { logJob } from "../../services/logging/log_job";
import { PageOptions, SearchOptions } from "../../lib/entities";
import { search } from "../../search";
@@ -23,6 +23,7 @@ import {
import { fromV0Combo } from "../v2/types";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq";
+import { defaultOrigin } from "../../lib/default-values";
async function searchHelper(
jobId: string,
@@ -117,7 +118,8 @@ async function searchHelper(
startTime: Date.now(),
zeroDataRetention: false, // not supported on v0
apiKeyId: api_key_id,
- },
+ origin: req.body.origin ?? defaultOrigin,
+ } satisfies ScrapeJobSingleUrls,
};
});
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index a4b5cd2bb..f0899c616 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -41,6 +41,9 @@ export async function crawlErrorsController(
res.status(200).json({
errors: failedJobs
.map(x => {
+ if (x.data.mode !== "single_urls") {
+ return null;
+ }
const error = deserializeTransportableError(
x.failedReason!,
) as TransportableError | null;
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index d856de2cf..07b079847 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -22,6 +22,7 @@ import { logger } from "../../lib/logger";
import { supabase_rr_service, supabase_service } from "../../services/supabase";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";
+import { ScrapeJobSingleUrls } from "../../types";
configDotenv();
export type PseudoJob<T> = {
@@ -47,7 +48,7 @@ export type DBJob = {
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [nuqJob, dbJob, gcsJob] = await Promise.all([
- scrapeQueue.getJob(id),
+ scrapeQueue.getJob(id) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,
(process.env.USE_DB_AUTHENTICATION === "true"
? supabaseGetJobById(id)
: null) as Promise<DBJob | null>,
@@ -58,6 +59,10 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
if (!nuqJob && !dbJob) return null;
+ if (nuqJob && nuqJob.data.mode !== "single_urls") {
+ return null;
+ }
+
const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;
if (gcsJob === null && data) {
logger.warn("GCS Job not found", {
@@ -83,7 +88,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([
- scrapeQueue.getJobs(ids),
+ scrapeQueue.getJobs(ids) as Promise<NuQJob<ScrapeJobSingleUrls>[]>,
process.env.USE_DB_AUTHENTICATION === "true"
? supabaseGetJobsById(ids)
: [],
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index c04236470..dfbd8597a 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -134,8 +134,8 @@ export async function crawlController(
await _addScrapeJobToBullMQ(
{
- url: req.body.url,
mode: "kickoff" as const,
+ url: req.body.url,
team_id: req.auth.team_id,
crawlerOptions,
scrapeOptions: sc.scrapeOptions,
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index baee96800..615a6537c 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -123,7 +123,7 @@ export async function scrapeController(
});
if (zeroDataRetention) {
- await scrapeQueue.removeJob(jobId);
+ await scrapeQueue.removeJob(jobId, logger);
}
if (e instanceof TransportableError) {
@@ -143,7 +143,7 @@ export async function scrapeController(
logger.info("Done with waitForJob");
- await scrapeQueue.removeJob(jobId);
+ await scrapeQueue.removeJob(jobId, logger);
logger.info("Removed job from queue");
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 00b51ed26..b4d3dd6e0 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -12,7 +12,6 @@ import { billTeam } from "../../services/billing/credit_billing";
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
-import { Mode } from "../../types";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import * as Sentry from "@sentry/node";
@@ -119,7 +118,7 @@ async function scrapeSearchResult(
await addScrapeJob(
{
url: searchResult.url,
- mode: "single_urls" as Mode,
+ mode: "single_urls",
team_id: options.teamId,
scrapeOptions: {
...scrapeOptions,
@@ -157,7 +156,7 @@ async function scrapeSearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await scrapeQueue.removeJob(jobId);
+ await scrapeQueue.removeJob(jobId, logger);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index 0e5aed0e0..10292e230 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -11,7 +11,6 @@ import {
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
-import { Mode } from "../../types";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import * as Sentry from "@sentry/node";
@@ -69,7 +68,7 @@ async function scrapeX402SearchResult(
await addScrapeJob(
{
url: searchResult.url,
- mode: "single_urls" as Mode,
+ mode: "single_urls",
team_id: options.teamId,
scrapeOptions: {
...scrapeOptions,
@@ -110,7 +109,7 @@ async function scrapeX402SearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await scrapeQueue.removeJob(jobId);
+ await scrapeQueue.removeJob(jobId, logger);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
index a4b5cd2bb..f0899c616 100644
--- a/apps/api/src/controllers/v2/crawl-errors.ts
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -41,6 +41,9 @@ export async function crawlErrorsController(
res.status(200).json({
errors: failedJobs
.map(x => {
+ if (x.data.mode !== "single_urls") {
+ return null;
+ }
const error = deserializeTransportableError(
x.failedReason!,
) as TransportableError | null;
diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.ts
index 6c4c6155b..91c1dd809 100644
--- a/apps/api/src/controllers/v2/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts
@@ -92,7 +92,10 @@ async function crawlStatusWS(
])
).map(x => x.id);
- const newlyDoneJobs: PseudoJob<any>[] = await getJobs(newlyDoneJobIDs);
+ const newlyDoneJobs: PseudoJob<any>[] = await getJobs(
+ newlyDoneJobIDs,
+ logger,
+ );
for (const job of newlyDoneJobs) {
if (job.returnvalue) {
@@ -145,7 +148,7 @@ async function crawlStatusWS(
jobIDs = validJobIDs; // Use validJobIDs instead of jobIDs for further processing
- const doneJobs = await getJobs(doneJobIDs);
+ const doneJobs = await getJobs(doneJobIDs, logger);
const data = doneJobs.map(x => x.returnvalue);
await send(ws, {
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 3520e2063..098b9cbdc 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -22,6 +22,7 @@ import { logger } from "../../lib/logger";
import { supabase_rr_service, supabase_service } from "../../services/supabase";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import { scrapeQueue, NuQJob, NuQJobStatus } from "../../services/worker/nuq";
+import { ScrapeJobSingleUrls } from "../../types";
configDotenv();
export type PseudoJob<T> = {
@@ -45,9 +46,15 @@ export type DBJob = {
team_id: string;
};
-export async function getJob(id: string): Promise<PseudoJob<any> | null> {
+export async function getJob(
+ id: string,
+ _logger = logger,
+): Promise<PseudoJob<any> | null> {
const [nuqJob, dbJob, gcsJob] = await Promise.all([
- scrapeQueue.getJob(id),
+ scrapeQueue.getJob(
+ id,
+ _logger,
+ ) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,
(process.env.USE_DB_AUTHENTICATION === "true"
? supabaseGetJobById(id)
: null) as Promise<DBJob | null>,
@@ -58,9 +65,13 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
if (!nuqJob && !dbJob) return null;
+ if (nuqJob && nuqJob.data.mode !== "single_urls") {
+ return null;
+ }
+
const data = gcsJob ?? dbJob?.docs ?? nuqJob?.returnvalue;
if (gcsJob === null && data) {
- logger.warn("GCS Job not found", {
+ _logger.warn("GCS Job not found", {
jobId: id,
});
}
@@ -81,9 +92,12 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
return job;
}
-export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
+export async function getJobs(
+ ids: string[],
+ _logger = logger,
+): Promise<PseudoJob<any>[]> {
const [nuqJobs, dbJobs, gcsJobs] = await Promise.all([
- scrapeQueue.getJobs(ids),
+ scrapeQueue.getJobs(ids, _logger) as Promise<NuQJob<ScrapeJobSingleUrls>[]>,
process.env.USE_DB_AUTHENTICATION === "true"
? supabaseGetJobsById(ids)
: [],
@@ -394,7 +408,7 @@ export async function crawlStatusController(
for (let i = 0; i < Math.ceil(doneJobs.length / 50); i++) {
const jobIds = doneJobs.slice(i * 50, (i + 1) * 50);
- const jobs = await getJobs(jobIds);
+ const jobs = await getJobs(jobIds, logger);
for (const job of jobs) {
if (job.status === "failed") {
diff --git a/apps/api/src/controllers/v2/scrape-status.ts b/apps/api/src/controllers/v2/scrape-status.ts
index 7127169ab..a1064b4a7 100644
--- a/apps/api/src/controllers/v2/scrape-status.ts
+++ b/apps/api/src/controllers/v2/scrape-status.ts
@@ -36,7 +36,7 @@ export async function scrapeStatusController(req: any, res: any) {
});
}
- const jobData = await getJob(req.params.jobId);
+ const jobData = await getJob(req.params.jobId, logger);
const data = Array.isArray(jobData?.returnvalue)
? jobData?.returnvalue[0]
: jobData?.returnvalue;
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index 3663f0135..1eff281e4 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -120,7 +120,7 @@ export async function scrapeController(
});
if (zeroDataRetention) {
- await scrapeQueue.removeJob(jobId);
+ await scrapeQueue.removeJob(jobId, logger);
}
if (e instanceof TransportableError) {
@@ -137,7 +137,7 @@ export async function scrapeController(
}
}
- await scrapeQueue.removeJob(jobId);
+ await scrapeQueue.removeJob(jobId, logger);
if (!hasFormatOfType(req.body.formats, "rawHtml")) {
if (doc && doc.rawHtml) {
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 8989abfa6..199ffae25 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -12,7 +12,6 @@ import { billTeam } from "../../services/billing/credit_billing";
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
-import { Mode } from "../../types";
import { search } from "../../search/v2";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import * as Sentry from "@sentry/node";
@@ -22,7 +21,7 @@ import { getJobPriority } from "../../lib/job-priority";
import { CostTracking } from "../../lib/cost-tracking";
import { calculateCreditsToBeBilled } from "../../lib/scrape-billing";
import { supabase_service } from "../../services/supabase";
-import { SearchResult, SearchV2Response } from "../../lib/entities";
+import { SearchV2Response } from "../../lib/entities";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq";
import { z } from "zod";
@@ -78,7 +77,7 @@ async function startScrapeJob(
await addScrapeJob(
{
url: searchResult.url,
- mode: "single_urls" as Mode,
+ mode: "single_urls",
team_id: options.teamId,
scrapeOptions: {
...options.scrapeOptions,
@@ -140,7 +139,7 @@ async function scrapeSearchResult(
origin: options.origin,
});
- await scrapeQueue.removeJob(jobId);
+ await scrapeQueue.removeJob(jobId, logger);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index 3e6a1019a..a740f444a 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -103,10 +103,12 @@ const logger = {
`${symbolColor}${symbol}${colors.reset} ${color}${colors.bold}${name}${colors.reset} ${timing}${codeInfo}`,
);
},
- processOutput(name: string, line: string) {
+ processOutput(name: string, line: string, isReduceNoise: boolean) {
const color = getProcessColor(name);
const label = `${color}${name.padEnd(12)}${colors.reset}`;
- console.log(`${label} ${line}`);
+ if (!(line.includes("[nuq/metrics:") && isReduceNoise)) {
+ console.log(`${label} ${line}`);
+ }
stream.write(`${name.padEnd(12)} ${line}\n`);
},
};
@@ -152,6 +154,9 @@ function execForward(
let displayCommand = "";
const isWindows = process.platform === "win32";
+ const isReduceNoise = env.NUQ_REDUCE_NOISE === "true";
+ delete env.NUQ_REDUCE_NOISE;
+
if (typeof command === "string") {
displayCommand = command;
if (isWindows) {
@@ -194,7 +199,7 @@ function execForward(
const remainingBuffer = lines[lines.length - 1];
completeLines.forEach(line => {
- if (line.trim()) logger.processOutput(name, line);
+ if (line.trim()) logger.processOutput(name, line, isReduceNoise);
});
if (isError) stderrBuffer = remainingBuffer;
@@ -317,6 +322,9 @@ function startServices(command?: string[]): Services {
process.argv[2] === "--start-docker"
? "node dist/src/index.js"
: "pnpm server:production:nobuild",
+ {
+ NUQ_REDUCE_NOISE: "true",
+ },
);
const worker = execForward(
@@ -324,6 +332,9 @@ function startServices(command?: string[]): Services {
process.argv[2] === "--start-docker"
? "node dist/src/services/queue-worker.js"
: "pnpm worker:production",
+ {
+ NUQ_REDUCE_NOISE: "true",
+ },
);
const nuqWorkers = Array.from({ length: 5 }, (_, i) =>
@@ -346,6 +357,9 @@ function startServices(command?: string[]): Services {
process.argv[2] === "--start-docker"
? "node dist/src/services/indexing/index-worker.js"
: "pnpm index-worker:production",
+ {
+ NUQ_REDUCE_NOISE: "true",
+ },
)
: undefined;
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
index 78ba0cda2..186835a68 100644
--- a/apps/api/src/lib/crawl-redis.ts
+++ b/apps/api/src/lib/crawl-redis.ts
@@ -197,7 +197,7 @@ async function isCrawlFinished(id: string) {
return (
(await redisEvictConnection.scard("crawl:" + id + ":jobs_done")) ===
(await redisEvictConnection.scard("crawl:" + id + ":jobs")) &&
- (await redisEvictConnection.get("crawl:" + id + ":kickoff:finish")) !== null
+ (await isCrawlKickoffFinished(id))
);
}
@@ -207,7 +207,10 @@ export async function isCrawlKickoffFinished(id: string) {
24 * 60 * 60,
);
return (
- (await redisEvictConnection.get("crawl:" + id + ":kickoff:finish")) !== null
+ (await redisEvictConnection.get("crawl:" + id + ":kickoff:finish")) !==
+ null &&
+ (await redisEvictConnection.scard("crawl:" + id + ":sitemap_jobs_done")) ===
+ (await redisEvictConnection.scard("crawl:" + id + ":sitemap_jobs"))
);
}
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index d7b70618e..e81582e0e 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -75,7 +75,7 @@ export async function scrapeDocument(
const doc = await waitForJob(jobId, timeout, false, logger);
try {
- await scrapeQueue.removeJob(jobId);
+ await scrapeQueue.removeJob(jobId, logger);
} catch (error) {
logger.warn("Error removing job from queue", { error, scrapeId: jobId });
}
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index df7319b8e..a1acbc8ec 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -74,7 +74,7 @@ export async function scrapeDocument_F0(
const doc = await waitForJob(jobId, timeout, false, logger);
try {
- await scrapeQueue.removeJob(jobId);
+ await scrapeQueue.removeJob(jobId, logger);
} catch (error) {
logger.warn("Error removing job from queue", { error, scrapeId: jobId });
}
diff --git a/apps/api/src/lib/logger.ts b/apps/api/src/lib/logger.ts
index 29d78e2d5..10a1e11a5 100644
--- a/apps/api/src/lib/logger.ts
+++ b/apps/api/src/lib/logger.ts
@@ -35,17 +35,6 @@ const zeroDataRetentionFilter = winston.format(info => {
return info;
})();
-const reduceNoiseFilter = winston.format(info => {
- if (
- (info.metadata?.module === "nuq/metrics" ||
- info.module === "nuq/metrics") &&
- process.env.NUQ_REDUCE_NOISE === "true"
- ) {
- return false; // Don't log this message
- }
- return info;
-})();
-
export const logger = winston.createLogger({
level: process.env.LOGGING_LEVEL?.toLowerCase() ?? "debug",
format: winston.format.json({
@@ -82,7 +71,6 @@ export const logger = winston.createLogger({
: []),
new winston.transports.Console({
format: winston.format.combine(
- reduceNoiseFilter,
zeroDataRetentionFilter,
winston.format.timestamp({ format: "YYYY-MM-DD HH:mm:ss" }),
winston.format.metadata({
diff --git a/apps/api/src/main/runWebScraper.ts b/apps/api/src/main/runWebScraper.ts
index 0ef187f87..c80c7dd3b 100644
--- a/apps/api/src/main/runWebScraper.ts
+++ b/apps/api/src/main/runWebScraper.ts
@@ -1,4 +1,4 @@
-import { WebScraperOptions, RunWebScraperParams } from "../types";
+import { ScrapeJobSingleUrls, RunWebScraperParams } from "../types";
import { logger as _logger } from "../lib/logger";
import { configDotenv } from "dotenv";
import { scrapeURL, ScrapeUrlResponse } from "../scraper/scrapeURL";
@@ -10,12 +10,11 @@ export async function startWebScraperPipeline({
job,
costTracking,
}: {
- job: NuQJob<WebScraperOptions>;
+ job: NuQJob<ScrapeJobSingleUrls>;
costTracking: CostTracking;
}) {
return await runWebScraper({
url: job.data.url,
- mode: job.data.mode,
scrapeOptions: {
...job.data.scrapeOptions,
...(job.data.crawl_id
@@ -34,7 +33,6 @@ export async function startWebScraperPipeline({
team_id: job.data.team_id,
bull_job_id: job.id,
priority: job.priority,
- is_scrape: job.data.is_scrape ?? false,
is_crawl: !!(job.data.crawl_id && job.data.crawlerOptions !== null),
urlInvisibleInCurrentCrawl:
job.data.crawlerOptions?.urlInvisibleInCurrentCrawl ?? false,
@@ -44,13 +42,11 @@ export async function startWebScraperPipeline({
async function runWebScraper({
url,
- mode,
scrapeOptions,
internalOptions,
team_id,
bull_job_id,
priority,
- is_scrape = false,
is_crawl = false,
urlInvisibleInCurrentCrawl = false,
costTracking,
diff --git a/apps/api/src/scraper/crawler/sitemap.ts b/apps/api/src/scraper/crawler/sitemap.ts
new file mode 100644
index 000000000..fafbc872b
--- /dev/null
+++ b/apps/api/src/scraper/crawler/sitemap.ts
@@ -0,0 +1,140 @@
+import type { Logger } from "winston";
+import { scrapeOptions, ScrapeOptions } from "../../controllers/v2/types";
+import { logger as _logger } from "../../lib/logger";
+import { Engine } from "../scrapeURL/engines";
+import { scrapeURL } from "../scrapeURL";
+import { CostTracking } from "../../lib/cost-tracking";
+import { processSitemap } from "@mendable/firecrawl-rs";
+import { fetchFileToBuffer } from "../scrapeURL/engines/utils/downloadFile";
+import { gunzip } from "node:zlib";
+import { promisify } from "node:util";
+
+const useFireEngine =
+ process.env.FIRE_ENGINE_BETA_URL !== "" &&
+ process.env.FIRE_ENGINE_BETA_URL !== undefined;
+
+type SitemapScrapeOptions = {
+ url: string;
+ maxAge: number;
+ zeroDataRetention: boolean;
+ location: ScrapeOptions["location"];
+ crawlId: string;
+ logger?: Logger;
+};
+
+type SitemapData = {
+ urls: URL[];
+ sitemaps: URL[];
+};
+
+const gunzipAsync = promisify(gunzip);
+
+async function _getSitemapXMLGZ(
+ options: SitemapScrapeOptions,
+): Promise<string> {
+ const { buffer } = await fetchFileToBuffer(options.url);
+ const decompressed = await gunzipAsync(buffer);
+ return decompressed.toString("utf-8");
+}
+
+async function getSitemapXML(options: SitemapScrapeOptions): Promise<string> {
+ if (options.url.toLowerCase().endsWith(".gz")) {
+ return await _getSitemapXMLGZ(options);
+ }
+
+ const isLocationSpecified =
+ options.location && options.location.country !== "us-generic";
+
+ const forceEngine: Engine[] = [
+ ...(options.maxAge > 0 ? ["index" as const] : []),
+ ...(isLocationSpecified && useFireEngine
+ ? [
+ "fire-engine;tlsclient" as const,
+ "fire-engine;tlsclient;stealth" as const,
+ ]
+ : []),
+ "fetch",
+ ...(!isLocationSpecified && useFireEngine
+ ? [
+ "fire-engine;tlsclient" as const,
+ "fire-engine;tlsclient;stealth" as const,
+ ]
+ : []),
+ ];
+
+ const response = await scrapeURL(
+ "sitemap;" + options.crawlId,
+ options.url,
+ scrapeOptions.parse({
+ formats: ["rawHtml"],
+ maxAge: options.maxAge,
+ ...(options.location ? { location: options.location } : {}),
+ }),
+ {
+ forceEngine,
+ v0DisableJsDom: true,
+ // externalAbort: options.abort,
+ teamId: "sitemap",
+ zeroDataRetention: options.zeroDataRetention,
+ crawlId: options.crawlId,
+ },
+ new CostTracking(),
+ );
+
+ if (
+ response.success &&
+ response.document.metadata.statusCode >= 200 &&
+ response.document.metadata.statusCode < 300
+ ) {
+ return response.document.rawHtml!;
+ } else if (!response.success) {
+ throw new Error("Failed to scrape sitemap", { cause: response.error });
+ } else {
+ throw new Error("Failed to scrape sitemap", {
+ cause: response.document.metadata.statusCode,
+ });
+ }
+}
+
+export async function scrapeSitemap(
+ options: SitemapScrapeOptions,
+): Promise<SitemapData> {
+ const logger = (options.logger ?? _logger).child({
+ module: "crawler",
+ method: "scrapeSitemap",
+ crawlId: options.crawlId,
+ sitemapUrl: options.url,
+ zeroDataRetention: options.zeroDataRetention,
+ });
+
+ logger.info("Scraping sitemap", {
+ maxAge: options.maxAge,
+ location: options.location,
+ });
+
+ const xml = await getSitemapXML(options);
+
+ logger.info("Processing sitemap");
+
+ const instructions = await processSitemap(xml);
+
+ const sitemapData: SitemapData = {
+ urls: [],
+ sitemaps: [],
+ };
+
+ for (const instruction of instructions.instructions) {
+ if (instruction.action === "recurse") {
+ sitemapData.sitemaps.push(...instruction.urls.map(url => new URL(url)));
+ } else if (instruction.action === "process") {
+ sitemapData.urls.push(...instruction.urls.map(url => new URL(url)));
+ }
+ }
+
+ logger.info("Processed sitemap", {
+ urls: sitemapData.urls.length,
+ sitemaps: sitemapData.sitemaps.length,
+ });
+
+ return sitemapData;
+}
diff --git a/apps/api/src/services/ab-test.ts b/apps/api/src/services/ab-test.ts
index 358ae65bd..97b252629 100644
--- a/apps/api/src/services/ab-test.ts
+++ b/apps/api/src/services/ab-test.ts
@@ -1,4 +1,4 @@
-import { WebScraperOptions } from "../types";
+import { ScrapeJobData } from "../types";
import { logger as _logger } from "../lib/logger";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import {
@@ -8,7 +8,7 @@ import {
FireEngineScrapeRequestTLSClient,
} from "../scraper/scrapeURL/engines/fire-engine/scrape";
-export function abTestJob(webScraperOptions: WebScraperOptions) {
+export function abTestJob(webScraperOptions: ScrapeJobData) {
// Global A/B test: mirror request to staging /v1/scrape based on SCRAPEURL_AB_RATE
const abLogger = _logger.child({ method: "ABTestToStaging" });
try {
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 67141224a..da0bda268 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -1,5 +1,5 @@
import { v4 as uuidv4 } from "uuid";
-import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";
+import { NotificationType, RateLimiterMode, ScrapeJobData } from "../types";
import {
cleanOldConcurrencyLimitEntries,
getConcurrencyLimitActiveJobs,
@@ -55,11 +55,13 @@ async function _addScrapeJobToConcurrencyQueue(
}
export async function _addScrapeJobToBullMQ(
- webScraperOptions: WebScraperOptions,
+ webScraperOptions: ScrapeJobData,
jobId: string,
priority: number = 0,
-): Promise<NuQJob<WebScraperOptions>> {
- abTestJob(webScraperOptions);
+): Promise<NuQJob<ScrapeJobData>> {
+ if (webScraperOptions.mode === "single_urls") {
+ abTestJob(webScraperOptions);
+ }
if (webScraperOptions && webScraperOptions.team_id) {
await pushConcurrencyLimitActiveJob(
@@ -70,7 +72,7 @@ export async function _addScrapeJobToBullMQ(
if (webScraperOptions.crawl_id) {
const sc = await getCrawl(webScraperOptions.crawl_id);
- if (webScraperOptions.crawlerOptions?.delay || sc?.maxConcurrency) {
+ if (sc?.crawlerOptions?.delay || sc?.maxConcurrency) {
await pushCrawlConcurrencyLimitActiveJob(
webScraperOptions.crawl_id,
jobId,
@@ -84,11 +86,11 @@ export async function _addScrapeJobToBullMQ(
}
async function addScrapeJobRaw(
- webScraperOptions: WebScraperOptions,
+ webScraperOptions: ScrapeJobData,
jobId: string,
priority: number = 0,
directToBullMQ: boolean = false,
-): Promise<NuQJob<WebScraperOptions> | null> {
+): Promise<NuQJob<ScrapeJobData> | null> {
let concurrencyLimited: "yes" | "yes-crawl" | "no" | null = null;
let currentActiveConcurrency = 0;
let maxConcurrency = 0;
@@ -124,7 +126,8 @@ async function addScrapeJobRaw(
webScraperOptions.team_id,
false,
true,
- webScraperOptions.is_extract
+ webScraperOptions.mode === "single_urls" &&
+ webScraperOptions.from_extract
? RateLimiterMode.Extract
: RateLimiterMode.Crawl,
)
@@ -181,11 +184,11 @@ async function addScrapeJobRaw(
}
export async function addScrapeJob(
- webScraperOptions: WebScraperOptions,
+ webScraperOptions: ScrapeJobData,
jobId: string = uuidv4(),
priority: number = 0,
directToBullMQ: boolean = false,
-): Promise<NuQJob<WebScraperOptions> | null> {
+): Promise<NuQJob<ScrapeJobData> | null> {
return await addScrapeJobRaw(
webScraperOptions,
jobId,
@@ -197,7 +200,7 @@ export async function addScrapeJob(
export async function addScrapeJobs(
jobs: {
jobId: string;
- data: WebScraperOptions;
+ data: ScrapeJobData;
priority: number;
}[],
) {
@@ -207,7 +210,7 @@ export async function addScrapeJobs(
string,
{
jobId: string;
- data: WebScraperOptions;
+ data: ScrapeJobData;
priority: number;
}[]
>();
@@ -222,13 +225,13 @@ export async function addScrapeJobs(
for (const [teamId, teamJobs] of jobsByTeam) {
// == Buckets for jobs ==
let jobsForcedToCQ: {
- data: WebScraperOptions;
+ data: ScrapeJobData;
jobId: string;
priority: number;
}[] = [];
let jobsPotentiallyInCQ: {
- data: WebScraperOptions;
+ data: ScrapeJobData;
jobId: string;
priority: number;
}[] = [];
@@ -237,14 +240,14 @@ export async function addScrapeJobs(
const jobsByCrawlID = new Map<
string,
{
- data: WebScraperOptions;
+ data: ScrapeJobData;
jobId: string;
priority: number;
}[]
>();
const jobsWithoutCrawlID: {
- data: WebScraperOptions;
+ data: ScrapeJobData;
jobId: string;
priority: number;
}[] = [];
@@ -297,7 +300,7 @@ export async function addScrapeJobs(
teamId,
false,
true,
- jobs[0].data.from_extract
+ jobs[0].data.mode === "single_urls" && jobs[0].data.from_extract
? RateLimiterMode.Extract
: RateLimiterMode.Crawl,
)
@@ -362,7 +365,7 @@ export async function addScrapeJobs(
}
export async function waitForJob(
- job: NuQJob<WebScraperOptions> | string,
+ job: NuQJob<ScrapeJobData> | string,
timeout: number | null,
zeroDataRetention: boolean,
logger: Logger = _logger,
@@ -374,7 +377,11 @@ export async function waitForJob(
try {
doc = await Promise.race(
[
- scrapeQueue.waitForJob(jobId, timeout !== null ? timeout + 100 : null),
+ scrapeQueue.waitForJob(
+ jobId,
+ timeout !== null ? timeout + 100 : null,
+ logger,
+ ),
timeout !== null
? new Promise<Document>((_resolve, reject) => {
setTimeout(() => {
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index 7d4eabab0..af3489d44 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -21,37 +21,47 @@ import { logJob } from "../logging/log_job";
import { createWebhookSender, WebhookEvent } from "../webhook";
import { hasFormatOfType } from "../../lib/format-utils";
import type { NuQJob } from "./nuq";
+import { ScrapeJobData } from "../../types";
+
+export async function finishCrawlIfNeeded(
+ job: NuQJob<ScrapeJobData>,
+ sc: StoredCrawl,
+) {
+ const crawlId = job.data.crawl_id!;
+
+ if (!crawlId) {
+ return;
+ }
-export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
const logger = _logger.child({
module: "queue-worker",
method: "finishCrawlIfNeeded",
jobId: job.id,
scrapeId: job.id,
- crawlId: job.data.crawl_id,
+ crawlId,
zeroDataRetention: sc.internalOptions.zeroDataRetention,
});
- if (await finishCrawlPre(job.data.crawl_id, logger)) {
+ if (await finishCrawlPre(crawlId, logger)) {
logger.info("Crawl is pre-finished, checking if we need to add more jobs");
if (
- job.data.crawlerOptions &&
+ sc.crawlerOptions &&
!(await redisEvictConnection.exists(
- "crawl:" + job.data.crawl_id + ":invisible_urls",
+ "crawl:" + crawlId + ":invisible_urls",
))
) {
await redisEvictConnection.set(
- "crawl:" + job.data.crawl_id + ":invisible_urls",
+ "crawl:" + crawlId + ":invisible_urls",
"done",
"EX",
60 * 60 * 24,
);
- const sc = (await getCrawl(job.data.crawl_id))!;
+ const sc = (await getCrawl(crawlId))!;
const visitedUrls = new Set(
await redisEvictConnection.smembers(
- "crawl:" + job.data.crawl_id + ":visited_unique",
+ "crawl:" + crawlId + ":visited_unique",
),
);
@@ -82,11 +92,11 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
});
const crawler = crawlToCrawler(
- job.data.crawl_id,
+ crawlId,
sc,
- (await getACUCTeam(job.data.team_id))?.flags ?? null,
+ (await getACUCTeam(job.data.team_id!))?.flags ?? null,
sc.originUrl!,
- job.data.crawlerOptions,
+ sc.crawlerOptions,
);
const univistedUrls = await crawler.filterLinks(
@@ -98,8 +108,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
const addableJobCount =
sc.crawlerOptions.limit === undefined
? Infinity
- : sc.crawlerOptions.limit -
- (await getDoneJobsOrderedLength(job.data.crawl_id));
+ : sc.crawlerOptions.limit - (await getDoneJobsOrderedLength(crawlId));
if (univistedUrls.links.length !== 0 && addableJobCount > 0) {
logger.info("Adding jobs", {
@@ -116,14 +125,14 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
mode: "single_urls" as const,
team_id: job.data.team_id,
crawlerOptions: {
- ...job.data.crawlerOptions,
+ ...sc.crawlerOptions,
urlInvisibleInCurrentCrawl: true,
},
- scrapeOptions: job.data.scrapeOptions,
+ scrapeOptions: sc.scrapeOptions,
internalOptions: sc.internalOptions,
origin: job.data.origin,
integration: job.data.integration,
- crawl_id: job.data.crawl_id,
+ crawl_id: crawlId,
sitemapped: true,
webhook: job.data.webhook,
v1: job.data.v1,
@@ -135,7 +144,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
});
const lockedIds = await lockURLsIndividually(
- job.data.crawl_id,
+ crawlId,
sc,
jobs.map(x => ({ id: x.jobId, url: x.data.url })),
);
@@ -143,7 +152,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
lockedIds.find(y => y.id === x.jobId),
);
await addCrawlJobs(
- job.data.crawl_id,
+ crawlId,
lockedJobs.map(x => x.jobId),
logger,
);
@@ -154,7 +163,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
lockedJobs: lockedJobs.length,
});
- await unPreFinishCrawl(job.data.crawl_id);
+ await unPreFinishCrawl(crawlId);
return;
} else {
logger.info(
@@ -165,10 +174,10 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
}
logger.info("Finishing crawl");
- await finishCrawl(job.data.crawl_id, logger);
+ await finishCrawl(crawlId, logger);
if (!job.data.v1) {
- const jobIDs = await getCrawlJobs(job.data.crawl_id);
+ const jobIDs = await getCrawlJobs(crawlId);
const jobs = (await getJobs(jobIDs)).sort(
(a, b) => a.timestamp - b.timestamp,
@@ -190,44 +199,30 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
await logJob(
{
- job_id: job.data.crawl_id,
+ job_id: crawlId,
success: jobStatus === "completed",
message: sc.cancelled ? "Cancelled" : undefined,
num_docs: fullDocs.length,
docs: [],
time_taken: (Date.now() - sc.createdAt) / 1000,
team_id: job.data.team_id,
- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
+ mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",
url: sc.originUrl!,
scrapeOptions: sc.scrapeOptions,
crawlerOptions: sc.crawlerOptions,
- origin: job.data.origin,
+ origin: sc.originUrl!,
integration: job.data.integration,
zeroDataRetention: job.data.zeroDataRetention,
},
false,
- job.data.internalOptions?.bypassBilling ?? false,
+ sc.internalOptions?.bypassBilling ?? false,
);
- const data = {
- success: jobStatus !== "failed",
- result: {
- links: fullDocs.map(doc => {
- return {
- content: doc,
- source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
- };
- }),
- },
- project_id: job.data.project_id,
- docs: fullDocs,
- };
-
// v0 web hooks, call when done with all the data
if (!job.data.v1) {
const sender = await createWebhookSender({
teamId: job.data.team_id,
- jobId: job.data.crawl_id,
+ jobId: crawlId,
webhook: job.data.webhook,
v0: true,
});
@@ -240,7 +235,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
},
source: doc?.metadata?.sourceURL ?? doc?.url ?? "",
}));
- if (job.data.crawlerOptions !== null) {
+ if (sc.crawlerOptions !== null) {
sender.send(WebhookEvent.CRAWL_COMPLETED, {
success: true,
data: documents,
@@ -254,7 +249,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
}
}
} else {
- const num_docs = await getDoneJobsOrderedLength(job.data.crawl_id);
+ const num_docs = await getDoneJobsOrderedLength(crawlId);
const jobStatus = sc.cancelled ? "failed" : "completed";
let credits_billed = null;
@@ -263,7 +258,7 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
const creditsRpc = await supabase_service.rpc(
"credits_billed_by_crawl_id_1",
{
- i_crawl_id: job.data.crawl_id,
+ i_crawl_id: crawlId,
},
);
@@ -278,18 +273,18 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
await logJob(
{
- job_id: job.data.crawl_id,
+ job_id: crawlId,
success: jobStatus === "completed",
message: sc.cancelled ? "Cancelled" : undefined,
num_docs,
docs: [],
time_taken: (Date.now() - sc.createdAt) / 1000,
- team_id: job.data.team_id,
+ team_id: sc.team_id,
scrapeOptions: sc.scrapeOptions,
- mode: job.data.crawlerOptions !== null ? "crawl" : "batch_scrape",
+ mode: sc.crawlerOptions !== null ? "crawl" : "batch_scrape",
url:
sc?.originUrl ??
- (job.data.crawlerOptions === null ? "Batch Scrape" : "Unknown"),
+ (sc.crawlerOptions === null ? "Batch Scrape" : "Unknown"),
crawlerOptions: sc.crawlerOptions,
origin: job.data.origin,
integration: job.data.integration,
@@ -297,19 +292,19 @@ export async function finishCrawlIfNeeded(job: NuQJob<any>, sc: StoredCrawl) {
zeroDataRetention: job.data.zeroDataRetention,
},
true,
- job.data.internalOptions?.bypassBilling ?? false,
+ sc.internalOptions?.bypassBilling ?? false,
);
// v1 web hooks, call when done with no data, but with event completed
if (job.data.v1 && job.data.webhook) {
const sender = await createWebhookSender({
teamId: job.data.team_id,
- jobId: job.data.crawl_id,
+ jobId: crawlId,
webhook: job.data.webhook,
v0: false,
});
if (sender) {
- if (job.data.crawlerOptions !== null) {
+ if (sc.crawlerOptions !== null) {
sender.send(WebhookEvent.CRAWL_COMPLETED, {
success: true,
data: [],
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index 993603a7a..d7dedde3b 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -1,6 +1,7 @@
import { Logger } from "winston";
import { logger } from "../../lib/logger";
import { Client, Pool } from "pg";
+import { type ScrapeJobData } from "../../types";
// === Basics
@@ -142,6 +143,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
public async getJob(
id: string,
+ _logger = logger,
): Promise<NuQJob<JobData, JobReturnValue> | null> {
const start = Date.now();
try {
@@ -154,7 +156,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
).rows[0],
);
} finally {
- logger.info("nuqGetJob metrics", {
+ _logger.info("nuqGetJob metrics", {
module: "nuq/metrics",
method: "nuqGetJob",
duration: Date.now() - start,
@@ -165,6 +167,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
public async getJobs(
ids: string[],
+ _logger: Logger = logger,
): Promise<NuQJob<JobData, JobReturnValue>[]> {
if (ids.length === 0) return [];
@@ -177,7 +180,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
)
).rows.map(row => this.rowToJob(row)!);
} finally {
- logger.info("nuqGetJobs metrics", {
+ _logger.info("nuqGetJobs metrics", {
module: "nuq/metrics",
method: "nuqGetJobs",
duration: Date.now() - start,
@@ -215,6 +218,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
public async getJobsWithStatuses(
ids: string[],
statuses: NuQJobStatus[],
+ _logger: Logger = logger,
): Promise<NuQJob<JobData, JobReturnValue>[]> {
if (ids.length === 0) return [];
@@ -227,7 +231,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
)
).rows.map(row => this.rowToJob(row)!);
} finally {
- logger.info("nuqGetJobsWithStatuses metrics", {
+ _logger.info("nuqGetJobsWithStatuses metrics", {
module: "nuq/metrics",
method: "nuqGetJobsWithStatuses",
duration: Date.now() - start,
@@ -237,7 +241,10 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
}
- public async removeJob(id: string): Promise<boolean> {
+ public async removeJob(
+ id: string,
+ _logger: Logger = logger,
+ ): Promise<boolean> {
const start = Date.now();
try {
return (
@@ -248,7 +255,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
).rowCount !== 0
);
} finally {
- logger.info("nuqRemoveJob metrics", {
+ _logger.info("nuqRemoveJob metrics", {
module: "nuq/metrics",
method: "nuqRemoveJob",
duration: Date.now() - start,
@@ -257,7 +264,10 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
}
- public async removeJobs(ids: string[]): Promise<number> {
+ public async removeJobs(
+ ids: string[],
+ _logger: Logger = logger,
+ ): Promise<number> {
if (ids.length === 0) return 0;
const start = Date.now();
@@ -271,7 +281,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
).rowCount ?? 0
);
} finally {
- logger.info("nuqRemoveJobs metrics", {
+ _logger.info("nuqRemoveJobs metrics", {
module: "nuq/metrics",
method: "nuqRemoveJobs",
duration: Date.now() - start,
@@ -315,6 +325,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
public waitForJob(
id: string,
timeout: number | null,
+ _logger: Logger = logger,
): Promise<JobReturnValue> {
const done = new Promise<JobReturnValue>(
(async (resolve, reject) => {
@@ -332,7 +343,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
const listener = async function (_msg: "completed" | "failed") {
if (timer) clearTimeout(timer);
- const job = await this.getJob(id);
+ const job = await this.getJob(id, _logger);
if (!job) {
reject(new Error("Job raced out while waiting for it"));
} else {
@@ -351,7 +362,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
}
try {
- const job = await this.getJob(id);
+ const job = await this.getJob(id, _logger);
if (job && ["completed", "failed"].includes(job.status)) {
this.removeListener(id, listener);
if (timer) clearTimeout(timer);
@@ -363,7 +374,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
return;
}
} catch (e) {
- logger.warn("nuqGetJob ensure check failed", {
+ _logger.warn("nuqGetJob ensure check failed", {
module: "nuq",
method: "nuqWaitForJob",
error: e,
@@ -374,7 +385,7 @@ class NuQ<JobData = any, JobReturnValue = any> {
const timeoutAt = timeout !== null ? Date.now() + timeout : null;
const poll = async function poll() {
try {
- const job = await this.getJob(id);
+ const job = await this.getJob(id, _logger);
if (job && ["completed", "failed"].includes(job.status)) {
if (job.status === "completed") {
return resolve(job.returnvalue!);
@@ -558,7 +569,7 @@ export async function nuqHealthCheck(): Promise<boolean> {
// === Instances
-export const scrapeQueue = new NuQ("nuq.queue_scrape");
+export const scrapeQueue = new NuQ<ScrapeJobData>("nuq.queue_scrape");
// === Cleanup
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index e40f44c69..72dff9795 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -25,7 +25,12 @@ import {
saveCrawl,
StoredCrawl,
} from "../../lib/crawl-redis";
-import { addScrapeJob, addScrapeJobs } from "../queue-jobs";
+import {
+ _addScrapeJobToBullMQ,
+ addScrapeJob,
+ addScrapeJobs,
+} from "../queue-jobs";
+import psl from "psl";
import { getJobPriority } from "../../lib/job-priority";
import { Document, scrapeOptions, TeamFlags } from "../../controllers/v2/types";
import { hasFormatOfType } from "../../lib/format-utils";
@@ -60,6 +65,13 @@ import { OTLPTraceExporter } from "@opentelemetry/exporter-trace-otlp-grpc";
import { resourceFromAttributes } from "@opentelemetry/resources";
import { ATTR_SERVICE_NAME } from "@opentelemetry/semantic-conventions";
import type { NuQJob } from "./nuq";
+import {
+ ScrapeJobData,
+ ScrapeJobKickoff,
+ ScrapeJobKickoffSitemap,
+ ScrapeJobSingleUrls,
+} from "../../types";
+import { scrapeSitemap } from "../../scraper/crawler/sitemap";
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
@@ -138,7 +150,7 @@ async function billScrapeJob(
return creditsToBeBilled;
}
-async function processJob(job: NuQJob<any>) {
+async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
const logger = _logger.child({
module: "queue-worker",
method: "processJob",
@@ -224,7 +236,6 @@ async function processJob(job: NuQJob<any>) {
},
],
},
- project_id: job.data.project_id,
document: doc,
};
@@ -441,7 +452,7 @@ async function processJob(job: NuQJob<any>) {
job.data.internalOptions?.bypassBilling ?? false,
);
- if (job.data.mode !== "crawl" && job.data.v1) {
+ if (job.data.v1) {
const sender = await createWebhookSender({
teamId: job.data.team_id,
jobId: job.data.crawl_id,
@@ -574,7 +585,6 @@ async function processJob(job: NuQJob<any>) {
const data = {
success: false,
document: null,
- project_id: job.data.project_id,
error:
error instanceof Error
? error
@@ -583,7 +593,7 @@ async function processJob(job: NuQJob<any>) {
: new Error(JSON.stringify(error)),
};
- if (job.data.mode === "crawl" || job.data.crawl_id) {
+ if (job.data.crawl_id) {
const sender = await createWebhookSender({
teamId: job.data.team_id,
jobId: (job.data.crawl_id ?? job.id) as string,
@@ -696,7 +706,55 @@ async function kickoffGetIndexLinks(
return validIndexLinks;
}
-async function processKickoffJob(job: NuQJob<any>) {
+async function addKickoffSitemapJob(
+ sitemapUrl: string,
+ sourceJob: NuQJob<ScrapeJobKickoff | ScrapeJobKickoffSitemap>,
+ sc: StoredCrawl,
+ logger: Logger,
+) {
+ const sitemapLocked =
+ (await redisEvictConnection.sadd(
+ "crawl:" + sourceJob.data.crawl_id + ":sitemaps",
+ sitemapUrl,
+ )) === 1;
+ await redisEvictConnection.expire(
+ "crawl:" + sourceJob.data.crawl_id + ":sitemaps",
+ 24 * 60 * 60,
+ );
+ if (!sitemapLocked) {
+ logger.debug("Sitemap already hit, skipping...", { sitemap: sitemapUrl });
+ return;
+ }
+
+ const jobId = uuidv4();
+ await _addScrapeJobToBullMQ(
+ {
+ mode: "kickoff_sitemap" as const,
+ team_id: sourceJob.data.team_id,
+ zeroDataRetention:
+ sourceJob.data.zeroDataRetention || (sc.zeroDataRetention ?? false),
+ sitemapUrl: sitemapUrl,
+ origin: sourceJob.data.origin,
+ integration: sourceJob.data.integration,
+ crawl_id: sourceJob.data.crawl_id,
+ webhook: sourceJob.data.webhook,
+ v1: sourceJob.data.v1,
+ apiKeyId: sourceJob.data.apiKeyId,
+ } satisfies ScrapeJobKickoffSitemap,
+ jobId,
+ 21,
+ );
+ await redisEvictConnection.sadd(
+ "crawl:" + sourceJob.data.crawl_id + ":sitemap_jobs",
+ jobId,
+ );
+ await redisEvictConnection.expire(
+ "crawl:" + sourceJob.data.crawl_id + ":sitemap_jobs",
+ 24 * 60 * 60,
+ );
+}
+
+async function processKickoffJob(job: NuQJob<ScrapeJobKickoff>) {
const logger = _logger.child({
module: "queue-worker",
method: "processKickoffJob",
@@ -757,68 +815,36 @@ async function processKickoffJob(job: NuQJob<any>) {
}
}
- const sitemap = sc.crawlerOptions.ignoreSitemap
- ? 0
- : await crawler.tryGetSitemap(async urls => {
- if (urls.length === 0) return;
-
- logger.debug("Using sitemap chunk of length " + urls.length, {
- sitemapLength: urls.length,
- });
-
- let jobPriority = await getJobPriority({
- team_id: job.data.team_id,
- basePriority: 21,
- });
- logger.debug("Using job priority " + jobPriority, { jobPriority });
-
- const jobs = urls.map(url => {
- const uuid = uuidv4();
- return {
- data: {
- url,
- mode: "single_urls" as const,
- team_id: job.data.team_id,
- crawlerOptions: job.data.crawlerOptions,
- scrapeOptions: job.data.scrapeOptions,
- internalOptions: sc.internalOptions,
- origin: job.data.origin,
- integration: job.data.integration,
- crawl_id: job.data.crawl_id,
- sitemapped: true,
- webhook: job.data.webhook,
- v1: job.data.v1,
- zeroDataRetention: job.data.zeroDataRetention,
- apiKeyId: job.data.apiKeyId,
- },
- jobId: uuid,
- priority: jobPriority,
- };
- });
-
- logger.debug("Locking URLs...");
- const lockedIds = await lockURLsIndividually(
- job.data.crawl_id,
- sc,
- jobs.map(x => ({ id: x.jobId, url: x.data.url })),
- );
- const lockedJobs = jobs.filter(x =>
- lockedIds.find(y => y.id === x.jobId),
- );
- logger.debug("Adding scrape jobs to Redis...");
- await addCrawlJobs(
- job.data.crawl_id,
- lockedJobs.map(x => x.jobId),
- logger,
- );
- logger.debug("Adding scrape jobs to BullMQ...");
- await addScrapeJobs(lockedJobs);
- });
-
- if (sitemap === 0) {
- logger.debug("Sitemap not found or ignored.", {
- ignoreSitemap: sc.crawlerOptions.ignoreSitemap,
- });
+ if (!sc.crawlerOptions.ignoreSitemap) {
+ if (job.data.url.endsWith(".xml") || job.data.url.endsWith(".xml.gz")) {
+ await addKickoffSitemapJob(job.data.url, job, sc, logger);
+ } else {
+ const urlObj = new URL(job.data.url);
+
+ const attempts: string[] = crawler.robots.getSitemaps();
+
+ // Append sitemap.xml
+ const urlWithSitemap = new URL(urlObj.href);
+ urlWithSitemap.pathname =
+ urlWithSitemap.pathname +
+ (urlObj.pathname.endsWith("/") ? "" : "/") +
+ "sitemap.xml";
+ urlWithSitemap.search = "";
+ urlWithSitemap.hash = "";
+ attempts.push(urlWithSitemap.href);
+
+ // Base sitemap.xml
+ attempts.push(new URL("/sitemap.xml", urlObj.href).href);
+
+ // Root domain sitemap.xml
+ const urlRootSitemap = new URL("/sitemap.xml", urlObj.href);
+ urlRootSitemap.hostname = psl.parse(urlObj.hostname).domain;
+ attempts.push(urlRootSitemap.href);
+
+ for (const attempt of attempts) {
+ await addKickoffSitemapJob(attempt, job, sc, logger);
+ }
+ }
}
const indexLinks = await kickoffGetIndexLinks(sc, crawler, job.data.url);
@@ -894,7 +920,130 @@ async function processKickoffJob(job: NuQJob<any>) {
}
}
-export const processJobInternal = async (job: NuQJob<any>) => {
+async function processKickoffSitemapJob(job: NuQJob<ScrapeJobKickoffSitemap>) {
+ const logger = _logger.child({
+ module: "queue-worker",
+ method: "processKickoffSitemapJob",
+ jobId: job.id,
+ scrapeId: job.id,
+ crawlId: job.data.crawl_id,
+ zeroDataRetention: job.data.zeroDataRetention ?? false,
+ });
+
+ const sc = await getCrawl(job.data.crawl_id);
+
+ try {
+ if (!sc) {
+ logger.error("Crawl not found");
+ return { success: false, error: "Crawl not found" };
+ }
+
+ const crawler = crawlToCrawler(
+ job.data.crawl_id,
+ sc,
+ (await getACUCTeam(job.data.team_id))?.flags ?? null,
+ );
+
+ const results = await scrapeSitemap({
+ url: job.data.sitemapUrl,
+ maxAge: 48 * 60 * 60 * 1000,
+ zeroDataRetention: job.data.zeroDataRetention ?? false,
+ location: job.data.location,
+ crawlId: job.data.crawl_id,
+ logger,
+ });
+
+ const passingURLs = (
+ await crawler.filterLinks(
+ [...new Set(results.urls.map(x => x.href))].filter(
+ url => crawler.filterURL(url, sc.originUrl!).allowed,
+ ),
+ Infinity,
+ sc.crawlerOptions.maxDepth ?? 10,
+ false,
+ )
+ ).links;
+
+ if (passingURLs.length > 0) {
+ logger.debug("Using urls of length " + passingURLs.length, {
+ urlsLength: passingURLs.length,
+ });
+
+ const jobPriority = await getJobPriority({
+ team_id: job.data.team_id,
+ basePriority: 21,
+ });
+
+ const jobs = passingURLs.map(url => ({
+ data: {
+ url: url,
+ mode: "single_urls" as const,
+ team_id: job.data.team_id,
+ crawlerOptions: sc.crawlerOptions,
+ scrapeOptions: sc.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: job.data.origin,
+ integration: job.data.integration,
+ crawl_id: job.data.crawl_id,
+ sitemapped: true,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ zeroDataRetention:
+ job.data.zeroDataRetention || (sc.zeroDataRetention ?? false),
+ apiKeyId: job.data.apiKeyId,
+ } satisfies ScrapeJobSingleUrls,
+ jobId: uuidv4(),
+ priority: jobPriority,
+ }));
+
+ const urls = await lockURLsIndividually(
+ job.data.crawl_id,
+ sc,
+ jobs.map(x => ({ id: x.jobId, url: x.data.url })),
+ );
+ const winningIds = new Set(urls.map(x => x.id));
+ await addCrawlJobs(
+ job.data.crawl_id,
+ urls.map(x => x.id),
+ logger,
+ );
+ await addScrapeJobs(jobs.filter(x => winningIds.has(x.jobId)));
+
+ logger.debug("Done queueing jobs!");
+ }
+
+ if (results.sitemaps.length > 0) {
+ logger.debug("Using sitemaps of length " + results.sitemaps.length, {
+ sitemapsLength: results.sitemaps.length,
+ });
+
+ for (const sitemap of results.sitemaps) {
+ await addKickoffSitemapJob(sitemap.href, job, sc, logger);
+ }
+
+ logger.debug("Done queueing sitemap jobs!");
+ }
+ return { success: true };
+ } catch (error) {
+ logger.error("An error occurred!", { error });
+ return { success: false, error };
+ } finally {
+ await redisEvictConnection.sadd(
+ "crawl:" + job.data.crawl_id + ":sitemap_jobs_done",
+ job.id,
+ );
+ await redisEvictConnection.expire(
+ "crawl:" + job.data.crawl_id + ":sitemap_jobs_done",
+ 24 * 60 * 60,
+ );
+
+ if (sc) {
+ await finishCrawlIfNeeded(job, sc);
+ }
+ }
+}
+
+export const processJobInternal = async (job: NuQJob<ScrapeJobData>) => {
const logger = _logger.child({
module: "queue-worker",
method: "processJobInternal",
@@ -919,15 +1068,26 @@ export const processJobInternal = async (job: NuQJob<any>) => {
await addJobPriority(job.data.team_id, job.id);
try {
- if (job.data?.mode === "kickoff") {
- const result = await processKickoffJob(job);
+ if (job.data.mode === "kickoff") {
+ const result = await processKickoffJob(
+ job as NuQJob<ScrapeJobKickoff>,
+ );
+ if (result.success) {
+ return null;
+ } else {
+ throw (result as any).error;
+ }
+ } else if (job.data.mode === "kickoff_sitemap") {
+ const result = await processKickoffSitemapJob(
+ job as NuQJob<ScrapeJobKickoffSitemap>,
+ );
if (result.success) {
return null;
} else {
throw (result as any).error;
}
} else {
- const result = await processJob(job);
+ const result = await processJob(job as NuQJob<ScrapeJobSingleUrls>);
if (result.success) {
try {
if (job.data.team_id) {
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index 28aced098..0690ec1e2 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -11,18 +11,27 @@ import { InternalOptions } from "./scraper/scrapeURL";
import type { CostTracking } from "./lib/cost-tracking";
import { webhookSchema } from "./services/webhook/schema";
-type Mode = "crawl" | "single_urls" | "sitemap" | "kickoff";
+type ScrapeJobCommon = {
+ concurrencyLimited?: boolean;
+ team_id: string;
+ zeroDataRetention: boolean;
+};
+
+export type ScrapeJobData = ScrapeJobCommon &
+ (
+ | ScrapeJobSingleUrlsUnique
+ | ScrapeJobKickoffUnique
+ | ScrapeJobKickoffSitemapUnique
+ );
-export { Mode };
+type ScrapeJobSingleUrlsUnique = {
+ mode: "single_urls";
-export interface WebScraperOptions {
url: string;
- mode: Mode;
crawlerOptions?: any;
scrapeOptions: BaseScrapeOptions;
internalOptions?: InternalOptions;
- team_id: string;
- origin?: string;
+ origin: string;
crawl_id?: string;
sitemapped?: boolean;
webhook?: z.infer<typeof webhookSchema>;
@@ -38,22 +47,53 @@ export interface WebScraperOptions {
from_extract?: boolean;
startTime?: number;
- zeroDataRetention: boolean;
sentry?: any;
is_extract?: boolean;
- concurrencyLimited?: boolean;
apiKeyId: number | null;
-}
+};
+
+export type ScrapeJobSingleUrls = ScrapeJobCommon & ScrapeJobSingleUrlsUnique;
+
+type ScrapeJobKickoffUnique = {
+ mode: "kickoff";
+
+ url: string;
+ crawlerOptions?: any;
+ scrapeOptions: BaseScrapeOptions;
+ internalOptions?: InternalOptions;
+ origin: string;
+ integration?: string | null;
+ crawl_id: string;
+ webhook?: z.infer<typeof webhookSchema>;
+ v1: boolean;
+ apiKeyId: number | null;
+};
+
+export type ScrapeJobKickoff = ScrapeJobCommon & ScrapeJobKickoffUnique;
+
+type ScrapeJobKickoffSitemapUnique = {
+ mode: "kickoff_sitemap";
+
+ crawl_id: string;
+ sitemapUrl: string;
+ location?: ScrapeOptions["location"];
+ origin: string;
+ integration?: string | null;
+ webhook?: z.infer<typeof webhookSchema>;
+ v1: boolean;
+ apiKeyId: number | null;
+};
+
+export type ScrapeJobKickoffSitemap = ScrapeJobCommon &
+ ScrapeJobKickoffSitemapUnique;
export interface RunWebScraperParams {
url: string;
- mode: Mode;
scrapeOptions: ScrapeOptions;
internalOptions?: InternalOptions;
team_id: string;
bull_job_id: string;
priority?: number;
- is_scrape?: boolean;
is_crawl?: boolean;
urlInvisibleInCurrentCrawl?: boolean;
costTracking: CostTracking;