Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Sitemap fixes (#1010)

Gergő Móricz0421f8102082
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index ceeaa4367..9659c218d 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -177,56 +177,51 @@ export async function crawlController(req: Request, res: Response) {
await saveCrawl(id, sc);
- const sitemap = sc.crawlerOptions?.ignoreSitemap
- ? null
- : await crawler.tryGetSitemap();
+ const sitemap = sc.crawlerOptions.ignoreSitemap
+ ? 0
+ : await crawler.tryGetSitemap(async urls => {
+ if (urls.length === 0) return;
+
+ let jobPriority = await getJobPriority({ plan, team_id, basePriority: 21 });
+ const jobs = urls.map(url => {
+ const uuid = uuidv4();
+ return {
+ name: uuid,
+ data: {
+ url,
+ mode: "single_urls",
+ crawlerOptions,
+ scrapeOptions,
+ internalOptions,
+ team_id,
+ plan,
+ origin: req.body.origin ?? defaultOrigin,
+ crawl_id: id,
+ sitemapped: true,
+ },
+ opts: {
+ jobId: uuid,
+ priority: jobPriority,
+ },
+ };
+ });
- if (sitemap !== null && sitemap.length > 0) {
- let jobPriority = 20;
- // If it is over 1000, we need to get the job priority,
- // otherwise we can use the default priority of 20
- if (sitemap.length > 1000) {
- // set base to 21
- jobPriority = await getJobPriority({ plan, team_id, basePriority: 21 });
- }
- const jobs = sitemap.map((x) => {
- const url = x.url;
- const uuid = uuidv4();
- return {
- name: uuid,
- data: {
- url,
- mode: "single_urls",
- crawlerOptions,
- scrapeOptions,
- internalOptions,
- team_id,
- plan,
- origin: req.body.origin ?? defaultOrigin,
- crawl_id: id,
- sitemapped: true,
- },
- opts: {
- jobId: uuid,
- priority: jobPriority,
- },
- };
- });
+ await lockURLs(
+ id,
+ sc,
+ jobs.map((x) => x.data.url),
+ );
+ await addCrawlJobs(
+ id,
+ jobs.map((x) => x.opts.jobId),
+ );
+ for (const job of jobs) {
+ // add with sentry instrumentation
+ await addScrapeJob(job.data as any, {}, job.opts.jobId);
+ }
+ });
- await lockURLs(
- id,
- sc,
- jobs.map((x) => x.data.url),
- );
- await addCrawlJobs(
- id,
- jobs.map((x) => x.opts.jobId),
- );
- for (const job of jobs) {
- // add with sentry instrumentation
- await addScrapeJob(job.data as any, {}, job.opts.jobId);
- }
- } else {
+ if (sitemap === 0) {
await lockURL(id, sc, url);
// Not needed, first one should be 15.
diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.ts
index f9462c3d2..9ba9bd463 100644
--- a/apps/api/src/controllers/v0/crawlPreview.ts
+++ b/apps/api/src/controllers/v0/crawlPreview.ts
@@ -113,32 +113,32 @@ export async function crawlPreviewController(req: Request, res: Response) {
const crawler = crawlToCrawler(id, sc);
const sitemap = sc.crawlerOptions?.ignoreSitemap
- ? null
- : await crawler.tryGetSitemap();
-
- if (sitemap !== null) {
- for (const url of sitemap.map((x) => x.url)) {
- await lockURL(id, sc, url);
- const jobId = uuidv4();
- await addScrapeJob(
- {
- url,
- mode: "single_urls",
- team_id,
- plan: plan!,
- crawlerOptions,
- scrapeOptions,
- internalOptions,
- origin: "website-preview",
- crawl_id: id,
- sitemapped: true,
- },
- {},
- jobId,
- );
- await addCrawlJob(id, jobId);
- }
- } else {
+ ? 0
+ : await crawler.tryGetSitemap(async urls => {
+ for (const url of urls) {
+ await lockURL(id, sc, url);
+ const jobId = uuidv4();
+ await addScrapeJob(
+ {
+ url,
+ mode: "single_urls",
+ team_id,
+ plan: plan!,
+ crawlerOptions,
+ scrapeOptions,
+ internalOptions,
+ origin: "website-preview",
+ crawl_id: id,
+ sitemapped: true,
+ },
+ {},
+ jobId,
+ );
+ await addCrawlJob(id, jobId);
+ }
+ });
+
+ if (sitemap === 0) {
await lockURL(id, sc, url);
const jobId = uuidv4();
await addScrapeJob(
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 59db16d82..1aec86c8e 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -115,7 +115,7 @@ export async function crawlStatusController(
const status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] =
sc.cancelled
? "cancelled"
- : validJobStatuses.every((x) => x[1] === "completed")
+ : (validJobStatuses.every((x) => x[1] === "completed") && validJobStatuses.length > 0)
? "completed"
: "scraping";
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index c2e3369f3..a759f4488 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -18,7 +18,7 @@ import {
} from "../../lib/crawl-redis";
import { logCrawl } from "../../services/logging/crawl_log";
import { getScrapeQueue } from "../../services/queue-service";
-import { addScrapeJob, addScrapeJobs } from "../../services/queue-jobs";
+import { _addScrapeJobToBullMQ, addScrapeJob, addScrapeJobs } from "../../services/queue-jobs";
import { logger as _logger } from "../../lib/logger";
import { getJobPriority } from "../../lib/job-priority";
import { callWebhook } from "../../services/webhook";
@@ -111,113 +111,20 @@ export async function crawlController(
await saveCrawl(id, sc);
- const sitemap = sc.crawlerOptions.ignoreSitemap
- ? null
- : await crawler.tryGetSitemap();
-
- if (sitemap !== null && sitemap.length > 0) {
- logger.debug("Using sitemap of length " + sitemap.length, {
- sitemapLength: sitemap.length,
- });
- let jobPriority = 20;
- // If it is over 1000, we need to get the job priority,
- // otherwise we can use the default priority of 20
- if (sitemap.length > 1000) {
- // set base to 21
- jobPriority = await getJobPriority({
- plan: req.auth.plan,
- team_id: req.auth.team_id,
- basePriority: 21,
- });
- }
- logger.debug("Using job priority " + jobPriority, { jobPriority });
-
- const jobs = sitemap.map((x) => {
- const url = x.url;
- const uuid = uuidv4();
- return {
- name: uuid,
- data: {
- url,
- mode: "single_urls" as const,
- team_id: req.auth.team_id,
- plan: req.auth.plan!,
- crawlerOptions,
- scrapeOptions,
- internalOptions: sc.internalOptions,
- origin: "api",
- crawl_id: id,
- sitemapped: true,
- webhook: req.body.webhook,
- v1: true,
- },
- opts: {
- jobId: uuid,
- priority: 20,
- },
- };
- });
-
- logger.debug("Locking URLs...");
- await lockURLs(
- id,
- sc,
- jobs.map((x) => x.data.url),
- );
- logger.debug("Adding scrape jobs to Redis...");
- await addCrawlJobs(
- id,
- jobs.map((x) => x.opts.jobId),
- );
- logger.debug("Adding scrape jobs to BullMQ...");
- await addScrapeJobs(jobs);
- } else {
- logger.debug("Sitemap not found or ignored.", {
- ignoreSitemap: sc.crawlerOptions.ignoreSitemap,
- });
-
- logger.debug("Locking URL...");
- await lockURL(id, sc, req.body.url);
- const jobId = uuidv4();
- logger.debug("Adding scrape job to Redis...", { jobId });
- await addScrapeJob(
- {
- url: req.body.url,
- mode: "single_urls",
- team_id: req.auth.team_id,
- crawlerOptions,
- scrapeOptions: scrapeOptionsSchema.parse(scrapeOptions),
- internalOptions: sc.internalOptions,
- plan: req.auth.plan!,
- origin: "api",
- crawl_id: id,
- webhook: req.body.webhook,
- v1: true,
- },
- {
- priority: 15,
- },
- jobId,
- );
- logger.debug("Adding scrape job to BullMQ...", { jobId });
- await addCrawlJob(id, jobId);
- }
- logger.debug("Done queueing jobs!");
-
- if (req.body.webhook) {
- logger.debug("Calling webhook with crawl.started...", {
- webhook: req.body.webhook,
- });
- await callWebhook(
- req.auth.team_id,
- id,
- null,
- req.body.webhook,
- true,
- "crawl.started",
- );
- }
-
+ await _addScrapeJobToBullMQ({
+ url: req.body.url,
+ mode: "kickoff" as const,
+ team_id: req.auth.team_id,
+ plan: req.auth.plan,
+ crawlerOptions,
+ scrapeOptions: sc.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: "api",
+ crawl_id: id,
+ webhook: req.body.webhook,
+ v1: true,
+ }, {}, crypto.randomUUID(), 10);
+
const protocol = process.env.ENV === "local" ? req.protocol : "https";
return res.status(200).json({
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index 27a926fc8..3274dd93f 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -86,11 +86,12 @@ export async function getMapResults({
// If sitemapOnly is true, only get links from sitemap
if (crawlerOptions.sitemapOnly) {
- const sitemap = await crawler.tryGetSitemap(true, true);
- if (sitemap !== null) {
- sitemap.forEach((x) => {
- links.push(x.url);
+ const sitemap = await crawler.tryGetSitemap(urls => {
+ urls.forEach((x) => {
+ links.push(x);
});
+ }, true, true);
+ if (sitemap > 0) {
links = links
.slice(1)
.map((x) => {
@@ -143,8 +144,10 @@ export async function getMapResults({
}
// Parallelize sitemap fetch with serper search
- const [sitemap, ...searchResults] = await Promise.all([
- ignoreSitemap ? null : crawler.tryGetSitemap(true),
+ const [_, ...searchResults] = await Promise.all([
+ ignoreSitemap ? null : crawler.tryGetSitemap(urls => {
+ links.push(...urls);
+ }, true),
...(cachedResult ? [] : pagePromises),
]);
@@ -152,12 +155,6 @@ export async function getMapResults({
allResults = searchResults;
}
- if (sitemap !== null) {
- sitemap.forEach((x) => {
- links.push(x.url);
- });
- }
-
mapResults = allResults
.flat()
.filter((result) => result !== null && result !== undefined);
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
index 41bee2d6d..52bea9e5b 100644
--- a/apps/api/src/scraper/WebScraper/crawler.ts
+++ b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -4,9 +4,10 @@ import { URL } from "url";
import { getLinksFromSitemap } from "./sitemap";
import robotsParser from "robots-parser";
import { getURLDepth } from "./utils/maxDepthUtils";
-import { axiosTimeout } from "../../../src/lib/timeout";
-import { logger as _logger } from "../../../src/lib/logger";
+import { axiosTimeout } from "../../lib/timeout";
+import { logger as _logger } from "../../lib/logger";
import https from "https";
+import { redisConnection } from "../../services/queue-service";
export class WebCrawler {
private jobId: string;
private initialUrl: string;
@@ -198,26 +199,60 @@ export class WebCrawler {
}
public async tryGetSitemap(
+ urlsHandler: (urls: string[]) => unknown,
fromMap: boolean = false,
onlySitemap: boolean = false,
- ): Promise<{ url: string; html: string }[] | null> {
+ ): Promise<number> {
this.logger.debug(`Fetching sitemap links from ${this.initialUrl}`, {
method: "tryGetSitemap",
});
- const sitemapLinks = await this.tryFetchSitemapLinks(this.initialUrl);
- if (fromMap && onlySitemap) {
- return sitemapLinks.map((link) => ({ url: link, html: "" }));
- }
- if (sitemapLinks.length > 0) {
- let filteredLinks = this.filterLinks(
- [...new Set(sitemapLinks)],
- this.limit,
- this.maxCrawledDepth,
- fromMap,
- );
- return filteredLinks.map((link) => ({ url: link, html: "" }));
+ let leftOfLimit = this.limit;
+
+ const normalizeUrl = (url: string) => {
+ url = url.replace(/^https?:\/\//, "").replace(/^www\./, "");
+ if (url.endsWith("/")) {
+ url = url.slice(0, -1);
+ }
+ return url;
+ };
+
+ const _urlsHandler = async (urls: string[]) => {
+ let uniqueURLs: string[] = [];
+ for (const url of urls) {
+ if (await redisConnection.sadd("sitemap:" + this.jobId + ":links", normalizeUrl(url))) {
+ uniqueURLs.push(url);
+ }
+ }
+
+ await redisConnection.expire("sitemap:" + this.jobId + ":links", 3600, "NX");
+ if (uniqueURLs.length > 0) {
+ urlsHandler(uniqueURLs);
+ }
+ };
+
+ let count = await this.tryFetchSitemapLinks(this.initialUrl, (urls: string[]) => {
+ if (fromMap && onlySitemap) {
+ return urlsHandler(urls);
+ } else {
+ let filteredLinks = this.filterLinks(
+ [...new Set(urls)],
+ leftOfLimit,
+ this.maxCrawledDepth,
+ fromMap,
+ );
+ leftOfLimit -= filteredLinks.length;
+ return _urlsHandler(filteredLinks);
+ }
+ });
+
+ if (count > 0) {
+ if (await redisConnection.sadd("sitemap:" + this.jobId + ":links", normalizeUrl(this.initialUrl))) {
+ urlsHandler([this.initialUrl]);
+ }
+ count++;
}
- return null;
+
+ return count;
}
public filterURL(href: string, url: string): string | null {
@@ -436,22 +471,15 @@ export class WebCrawler {
return socialMediaOrEmail.some((ext) => url.includes(ext));
}
- private async tryFetchSitemapLinks(url: string): Promise<string[]> {
- const normalizeUrl = (url: string) => {
- url = url.replace(/^https?:\/\//, "").replace(/^www\./, "");
- if (url.endsWith("/")) {
- url = url.slice(0, -1);
- }
- return url;
- };
-
+ private async tryFetchSitemapLinks(url: string, urlsHandler: (urls: string[]) => unknown): Promise<number> {
const sitemapUrl = url.endsWith(".xml") ? url : `${url}/sitemap.xml`;
- let sitemapLinks: string[] = [];
+
+ let sitemapCount: number = 0;
// Try to get sitemap from the provided URL first
try {
- sitemapLinks = await getLinksFromSitemap(
- { sitemapUrl, allUrls: [], mode: "fire-engine" },
+ sitemapCount = await getLinksFromSitemap(
+ { sitemapUrl, urlsHandler, mode: "fire-engine" },
this.logger,
);
} catch (error) {
@@ -476,20 +504,18 @@ export class WebCrawler {
try {
// Get all links from the main domain's sitemap
- const mainDomainLinks = await getLinksFromSitemap(
- { sitemapUrl: mainDomainSitemapUrl, allUrls: [], mode: "fire-engine" },
+ sitemapCount += await getLinksFromSitemap(
+ { sitemapUrl: mainDomainSitemapUrl, urlsHandler(urls) {
+ urlsHandler(urls.filter(link => {
+ try {
+ const linkUrl = new URL(link);
+ return linkUrl.hostname.endsWith(hostname);
+ } catch {
+ }
+ }))
+ }, mode: "fire-engine" },
this.logger,
);
- // Filter links to only include those pointing to the current subdomain
- const subdomainLinks = mainDomainLinks.filter(link => {
- try {
- const linkUrl = new URL(link);
- return linkUrl.hostname.endsWith(hostname);
- } catch {
- return false;
- }
- });
- sitemapLinks = [...new Set([...sitemapLinks, ...subdomainLinks])];
} catch (error) {
this.logger.debug(
`Failed to fetch main domain sitemap from ${mainDomainSitemapUrl}`,
@@ -506,15 +532,13 @@ export class WebCrawler {
}
// If no sitemap found yet, try the baseUrl as a last resort
- if (sitemapLinks.length === 0) {
+ if (sitemapCount === 0) {
const baseUrlSitemap = `${this.baseUrl}/sitemap.xml`;
try {
- const baseLinks = await getLinksFromSitemap(
- { sitemapUrl: baseUrlSitemap, allUrls: [], mode: "fire-engine" },
+ sitemapCount += await getLinksFromSitemap(
+ { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },
this.logger,
);
-
- sitemapLinks = [...new Set([...sitemapLinks, ...baseLinks])];
} catch (error) {
this.logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}`, {
method: "tryFetchSitemapLinks",
@@ -524,25 +548,14 @@ export class WebCrawler {
if (error instanceof AxiosError && error.response?.status === 404) {
// ignore 404
} else {
- sitemapLinks = await getLinksFromSitemap(
- { sitemapUrl: baseUrlSitemap, mode: "fire-engine" },
+ sitemapCount += await getLinksFromSitemap(
+ { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },
this.logger,
);
}
}
}
- const normalizedUrl = normalizeUrl(url);
- const normalizedSitemapLinks = sitemapLinks.map((link) =>
- normalizeUrl(link),
- );
- // has to be greater than 0 to avoid adding the initial URL to the sitemap links, and preventing crawler to crawl
- if (
- !normalizedSitemapLinks.includes(normalizedUrl) &&
- sitemapLinks.length > 0
- ) {
- sitemapLinks.push(url);
- }
- return sitemapLinks;
+ return sitemapCount;
}
}
diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.ts
index 2529c022d..8028d225c 100644
--- a/apps/api/src/scraper/WebScraper/sitemap.ts
+++ b/apps/api/src/scraper/WebScraper/sitemap.ts
@@ -11,15 +11,15 @@ const useFireEngine =
export async function getLinksFromSitemap(
{
sitemapUrl,
- allUrls = [],
+ urlsHandler,
mode = "axios",
}: {
sitemapUrl: string;
- allUrls?: string[];
+ urlsHandler(urls: string[]): unknown,
mode?: "axios" | "fire-engine";
},
logger: Logger,
-): Promise<string[]> {
+): Promise<number> {
try {
let content: string = "";
try {
@@ -31,9 +31,12 @@ export async function getLinksFromSitemap(
{ forceEngine: "fire-engine;tlsclient", v0DisableJsDom: true },
);
if (!response.success) {
- throw response.error;
+ logger.debug("Failed to scrape sitemap via TLSClient, falling back to axios...", { error: response.error })
+ const ar = await axios.get(sitemapUrl, { timeout: axiosTimeout });
+ content = ar.data;
+ } else {
+ content = response.document.rawHtml!;
}
- content = response.document.rawHtml!;
} else {
const response = await axios.get(sitemapUrl, { timeout: axiosTimeout });
content = response.data;
@@ -45,11 +48,13 @@ export async function getLinksFromSitemap(
sitemapUrl,
error,
});
- return allUrls;
+
+ return 0;
}
const parsed = await parseStringPromise(content);
const root = parsed.urlset || parsed.sitemapindex;
+ let count = 0;
if (root && root.sitemap) {
// Handle sitemap index files
@@ -57,20 +62,18 @@ export async function getLinksFromSitemap(
.filter((sitemap) => sitemap.loc && sitemap.loc.length > 0)
.map((sitemap) => sitemap.loc[0]);
- const sitemapPromises = sitemapUrls.map((sitemapUrl) =>
+ const sitemapPromises: Promise<number>[] = sitemapUrls.map((sitemapUrl) =>
getLinksFromSitemap(
- { sitemapUrl, allUrls: [], mode },
+ { sitemapUrl, urlsHandler, mode },
logger,
),
);
const results = await Promise.all(sitemapPromises);
- results.forEach(urls => {
- allUrls.push(...urls);
- });
+ count = results.reduce((a,x) => a + x)
} else if (root && root.url) {
// Check if any URLs point to additional sitemaps
- const xmlSitemaps = root.url
+ const xmlSitemaps: string[] = root.url
.filter(
(url) =>
url.loc &&
@@ -83,18 +86,13 @@ export async function getLinksFromSitemap(
// Recursively fetch links from additional sitemaps
const sitemapPromises = xmlSitemaps.map((sitemapUrl) =>
getLinksFromSitemap(
- { sitemapUrl, allUrls: [], mode },
+ { sitemapUrl: sitemapUrl, urlsHandler, mode },
logger,
),
);
-
- const results = await Promise.all(sitemapPromises);
- results.forEach(urls => {
- allUrls.push(...urls);
- });
+ count += (await Promise.all(sitemapPromises)).reduce((a,x) => a + x, 0);
}
- // Add regular URLs that aren't sitemaps
const validUrls = root.url
.filter(
(url) =>
@@ -104,8 +102,11 @@ export async function getLinksFromSitemap(
!WebCrawler.prototype.isFile(url.loc[0]),
)
.map((url) => url.loc[0]);
- allUrls.push(...validUrls);
+ count += validUrls.length;
+ urlsHandler(validUrls);
}
+
+ return count;
} catch (error) {
logger.debug(`Error processing sitemapUrl: ${sitemapUrl}`, {
method: "getLinksFromSitemap",
@@ -115,7 +116,7 @@ export async function getLinksFromSitemap(
});
}
- return [...new Set(allUrls)];
+ return 0;
}
export const fetchSitemapData = async (
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 654f6cda0..f59babe4f 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -29,7 +29,7 @@ async function _addScrapeJobToConcurrencyQueue(
});
}
-async function _addScrapeJobToBullMQ(
+export async function _addScrapeJobToBullMQ(
webScraperOptions: any,
options: any,
jobId: string,
@@ -138,7 +138,6 @@ export async function addScrapeJobs(
if (jobs[0].data && jobs[0].data.team_id && jobs[0].data.plan) {
const now = Date.now();
const limit = await getConcurrencyLimitMax(jobs[0].data.plan);
- console.log("CC limit", limit);
cleanOldConcurrencyLimitEntries(jobs[0].data.team_id, now);
countCanBeDirectlyAdded = Math.max(
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 4ef9610de..e8c8bdf30 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -18,16 +18,18 @@ import { v4 as uuidv4 } from "uuid";
import {
addCrawlJob,
addCrawlJobDone,
+ addCrawlJobs,
crawlToCrawler,
finishCrawl,
generateURLPermutations,
getCrawl,
getCrawlJobs,
lockURL,
+ lockURLs,
normalizeURL,
} from "../lib/crawl-redis";
import { StoredCrawl } from "../lib/crawl-redis";
-import { addScrapeJob } from "./queue-jobs";
+import { addScrapeJob, addScrapeJobs } from "./queue-jobs";
import {
addJobPriority,
deleteJobPriority,
@@ -191,22 +193,34 @@ const processJobInternal = async (token: string, job: Job & { id: string }) => {
await addJobPriority(job.data.team_id, job.id);
let err = null;
try {
- const result = await processJob(job, token);
- if (result.success) {
- try {
- if (job.data.crawl_id && process.env.USE_DB_AUTHENTICATION === "true") {
- logger.debug(
- "Job succeeded -- has crawl associated, putting null in Redis",
- );
+ if (job.data?.mode === "kickoff") {
+ const result = await processKickoffJob(job, token);
+ if (result.success) {
+ try {
await job.moveToCompleted(null, token, false);
- } else {
- logger.debug("Job succeeded -- putting result in Redis");
- await job.moveToCompleted(result.document, token, false);
- }
- } catch (e) {}
+ } catch (e) {}
+ } else {
+ logger.debug("Job failed", { result, mode: job.data.mode });
+ await job.moveToFailed((result as any).error, token, false);
+ }
} else {
- logger.debug("Job failed", { result });
- await job.moveToFailed((result as any).error, token, false);
+ const result = await processJob(job, token);
+ if (result.success) {
+ try {
+ if (job.data.crawl_id && process.env.USE_DB_AUTHENTICATION === "true") {
+ logger.debug(
+ "Job succeeded -- has crawl associated, putting null in Redis",
+ );
+ await job.moveToCompleted(null, token, false);
+ } else {
+ logger.debug("Job succeeded -- putting result in Redis");
+ await job.moveToCompleted(result.document, token, false);
+ }
+ } catch (e) {}
+ } else {
+ logger.debug("Job failed", { result });
+ await job.moveToFailed((result as any).error, token, false);
+ }
}
} catch (error) {
logger.debug("Job failed", { error });
@@ -379,6 +393,130 @@ const workerFun = async (
workerFun(getScrapeQueue(), processJobInternal);
+async function processKickoffJob(job: Job & { id: string }, token: string) {
+ const logger = _logger.child({
+ module: "queue-worker",
+ method: "processKickoffJob",
+ jobId: job.id,
+ scrapeId: job.id,
+ crawlId: job.data?.crawl_id ?? undefined,
+ teamId: job.data?.team_id ?? undefined,
+ });
+
+ try {
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+ const crawler = crawlToCrawler(job.data.crawl_id, sc);
+
+ const sitemap = sc.crawlerOptions.ignoreSitemap
+ ? 0
+ : await crawler.tryGetSitemap(async urls => {
+ if (urls.length === 0) return;
+
+ logger.debug("Using sitemap chunk of length " + urls.length, {
+ sitemapLength: urls.length,
+ });
+
+ let jobPriority = await getJobPriority({
+ plan: job.data.plan,
+ team_id: job.data.team_id,
+ basePriority: 21,
+ });
+ logger.debug("Using job priority " + jobPriority, { jobPriority });
+
+ const jobs = urls.map(url => {
+ const uuid = uuidv4();
+ return {
+ name: uuid,
+ data: {
+ url,
+ mode: "single_urls" as const,
+ team_id: job.data.team_id,
+ plan: job.data.plan!,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: job.data.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: job.data.origin,
+ crawl_id: job.data.crawl_id,
+ sitemapped: true,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ },
+ opts: {
+ jobId: uuid,
+ priority: 20,
+ },
+ };
+ });
+
+ logger.debug("Locking URLs...");
+ await lockURLs(
+ job.data.crawl_id,
+ sc,
+ jobs.map((x) => x.data.url),
+ );
+ logger.debug("Adding scrape jobs to Redis...");
+ await addCrawlJobs(
+ job.data.crawl_id,
+ jobs.map((x) => x.opts.jobId),
+ );
+ logger.debug("Adding scrape jobs to BullMQ...");
+ await addScrapeJobs(jobs);
+ });
+
+ if (sitemap === 0) {
+ logger.debug("Sitemap not found or ignored.", {
+ ignoreSitemap: sc.crawlerOptions.ignoreSitemap,
+ });
+
+ logger.debug("Locking URL...");
+ await lockURL(job.data.crawl_id, sc, job.data.url);
+ const jobId = uuidv4();
+ logger.debug("Adding scrape job to Redis...", { jobId });
+ await addScrapeJob(
+ {
+ url: job.data.url,
+ mode: "single_urls",
+ team_id: job.data.team_id,
+ crawlerOptions: job.data.crawlerOptions,
+ scrapeOptions: scrapeOptions.parse(job.data.scrapeOptions),
+ internalOptions: sc.internalOptions,
+ plan: job.data.plan!,
+ origin: job.data.origin,
+ crawl_id: job.data.crawl_id,
+ webhook: job.data.webhook,
+ v1: job.data.v1,
+ },
+ {
+ priority: 15,
+ },
+ jobId,
+ );
+ logger.debug("Adding scrape job to BullMQ...", { jobId });
+ await addCrawlJob(job.data.crawl_id, jobId);
+ }
+ logger.debug("Done queueing jobs!");
+
+ if (job.data.webhook) {
+ logger.debug("Calling webhook with crawl.started...", {
+ webhook: job.data.webhook,
+ });
+ await callWebhook(
+ job.data.team_id,
+ job.data.crawl_id,
+ null,
+ job.data.webhook,
+ true,
+ "crawl.started",
+ );
+ }
+
+ return { success: true }
+ } catch (error) {
+ logger.error("An error occurred!", { error })
+ return { success: false, error };
+ }
+}
+
async function processJob(job: Job & { id: string }, token: string) {
const logger = _logger.child({
module: "queue-worker",