Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Sitemap fixes (#1010)
Gergő Móricz0421f8102082
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex ceeaa4367..9659c218d 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -177,56 +177,51 @@ export async function crawlController(req: Request, res: Response) {await saveCrawl(id, sc);- const sitemap = sc.crawlerOptions?.ignoreSitemap- ? null- : await crawler.tryGetSitemap();+ const sitemap = sc.crawlerOptions.ignoreSitemap+ ? 0+ : await crawler.tryGetSitemap(async urls => {+ if (urls.length === 0) return;++ let jobPriority = await getJobPriority({ plan, team_id, basePriority: 21 });+ const jobs = urls.map(url => {+ const uuid = uuidv4();+ return {+ name: uuid,+ data: {+ url,+ mode: "single_urls",+ crawlerOptions,+ scrapeOptions,+ internalOptions,+ team_id,+ plan,+ origin: req.body.origin ?? defaultOrigin,+ crawl_id: id,+ sitemapped: true,+ },+ opts: {+ jobId: uuid,+ priority: jobPriority,+ },+ };+ });- if (sitemap !== null && sitemap.length > 0) {- let jobPriority = 20;- // If it is over 1000, we need to get the job priority,- // otherwise we can use the default priority of 20- if (sitemap.length > 1000) {- // set base to 21- jobPriority = await getJobPriority({ plan, team_id, basePriority: 21 });- }- const jobs = sitemap.map((x) => {- const url = x.url;- const uuid = uuidv4();- return {- name: uuid,- data: {- url,- mode: "single_urls",- crawlerOptions,- scrapeOptions,- internalOptions,- team_id,- plan,- origin: req.body.origin ?? defaultOrigin,- crawl_id: id,- sitemapped: true,- },- opts: {- jobId: uuid,- priority: jobPriority,- },- };- });+ await lockURLs(+ id,+ sc,+ jobs.map((x) => x.data.url),+ );+ await addCrawlJobs(+ id,+ jobs.map((x) => x.opts.jobId),+ );+ for (const job of jobs) {+ // add with sentry instrumentation+ await addScrapeJob(job.data as any, {}, job.opts.jobId);+ }+ });- await lockURLs(- id,- sc,- jobs.map((x) => x.data.url),- );- await addCrawlJobs(- id,- jobs.map((x) => x.opts.jobId),- );- for (const job of jobs) {- // add with sentry instrumentation- await addScrapeJob(job.data as any, {}, job.opts.jobId);- }- } else {+ if (sitemap === 0) {await lockURL(id, sc, url);// Not needed, first one should be 15.diff --git a/apps/api/src/controllers/v0/crawlPreview.ts b/apps/api/src/controllers/v0/crawlPreview.tsindex f9462c3d2..9ba9bd463 100644--- a/apps/api/src/controllers/v0/crawlPreview.ts+++ b/apps/api/src/controllers/v0/crawlPreview.ts@@ -113,32 +113,32 @@ export async function crawlPreviewController(req: Request, res: Response) {const crawler = crawlToCrawler(id, sc);const sitemap = sc.crawlerOptions?.ignoreSitemap- ? null- : await crawler.tryGetSitemap();-- if (sitemap !== null) {- for (const url of sitemap.map((x) => x.url)) {- await lockURL(id, sc, url);- const jobId = uuidv4();- await addScrapeJob(- {- url,- mode: "single_urls",- team_id,- plan: plan!,- crawlerOptions,- scrapeOptions,- internalOptions,- origin: "website-preview",- crawl_id: id,- sitemapped: true,- },- {},- jobId,- );- await addCrawlJob(id, jobId);- }- } else {+ ? 0+ : await crawler.tryGetSitemap(async urls => {+ for (const url of urls) {+ await lockURL(id, sc, url);+ const jobId = uuidv4();+ await addScrapeJob(+ {+ url,+ mode: "single_urls",+ team_id,+ plan: plan!,+ crawlerOptions,+ scrapeOptions,+ internalOptions,+ origin: "website-preview",+ crawl_id: id,+ sitemapped: true,+ },+ {},+ jobId,+ );+ await addCrawlJob(id, jobId);+ }+ });++ if (sitemap === 0) {await lockURL(id, sc, url);const jobId = uuidv4();await addScrapeJob(diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 59db16d82..1aec86c8e 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -115,7 +115,7 @@ export async function crawlStatusController(const status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] =sc.cancelled? "cancelled"- : validJobStatuses.every((x) => x[1] === "completed")+ : (validJobStatuses.every((x) => x[1] === "completed") && validJobStatuses.length > 0)? "completed": "scraping";diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex c2e3369f3..a759f4488 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -18,7 +18,7 @@ import {} from "../../lib/crawl-redis";import { logCrawl } from "../../services/logging/crawl_log";import { getScrapeQueue } from "../../services/queue-service";-import { addScrapeJob, addScrapeJobs } from "../../services/queue-jobs";+import { _addScrapeJobToBullMQ, addScrapeJob, addScrapeJobs } from "../../services/queue-jobs";import { logger as _logger } from "../../lib/logger";import { getJobPriority } from "../../lib/job-priority";import { callWebhook } from "../../services/webhook";@@ -111,113 +111,20 @@ export async function crawlController(await saveCrawl(id, sc);- const sitemap = sc.crawlerOptions.ignoreSitemap- ? null- : await crawler.tryGetSitemap();-- if (sitemap !== null && sitemap.length > 0) {- logger.debug("Using sitemap of length " + sitemap.length, {- sitemapLength: sitemap.length,- });- let jobPriority = 20;- // If it is over 1000, we need to get the job priority,- // otherwise we can use the default priority of 20- if (sitemap.length > 1000) {- // set base to 21- jobPriority = await getJobPriority({- plan: req.auth.plan,- team_id: req.auth.team_id,- basePriority: 21,- });- }- logger.debug("Using job priority " + jobPriority, { jobPriority });-- const jobs = sitemap.map((x) => {- const url = x.url;- const uuid = uuidv4();- return {- name: uuid,- data: {- url,- mode: "single_urls" as const,- team_id: req.auth.team_id,- plan: req.auth.plan!,- crawlerOptions,- scrapeOptions,- internalOptions: sc.internalOptions,- origin: "api",- crawl_id: id,- sitemapped: true,- webhook: req.body.webhook,- v1: true,- },- opts: {- jobId: uuid,- priority: 20,- },- };- });-- logger.debug("Locking URLs...");- await lockURLs(- id,- sc,- jobs.map((x) => x.data.url),- );- logger.debug("Adding scrape jobs to Redis...");- await addCrawlJobs(- id,- jobs.map((x) => x.opts.jobId),- );- logger.debug("Adding scrape jobs to BullMQ...");- await addScrapeJobs(jobs);- } else {- logger.debug("Sitemap not found or ignored.", {- ignoreSitemap: sc.crawlerOptions.ignoreSitemap,- });-- logger.debug("Locking URL...");- await lockURL(id, sc, req.body.url);- const jobId = uuidv4();- logger.debug("Adding scrape job to Redis...", { jobId });- await addScrapeJob(- {- url: req.body.url,- mode: "single_urls",- team_id: req.auth.team_id,- crawlerOptions,- scrapeOptions: scrapeOptionsSchema.parse(scrapeOptions),- internalOptions: sc.internalOptions,- plan: req.auth.plan!,- origin: "api",- crawl_id: id,- webhook: req.body.webhook,- v1: true,- },- {- priority: 15,- },- jobId,- );- logger.debug("Adding scrape job to BullMQ...", { jobId });- await addCrawlJob(id, jobId);- }- logger.debug("Done queueing jobs!");-- if (req.body.webhook) {- logger.debug("Calling webhook with crawl.started...", {- webhook: req.body.webhook,- });- await callWebhook(- req.auth.team_id,- id,- null,- req.body.webhook,- true,- "crawl.started",- );- }-+ await _addScrapeJobToBullMQ({+ url: req.body.url,+ mode: "kickoff" as const,+ team_id: req.auth.team_id,+ plan: req.auth.plan,+ crawlerOptions,+ scrapeOptions: sc.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: "api",+ crawl_id: id,+ webhook: req.body.webhook,+ v1: true,+ }, {}, crypto.randomUUID(), 10);+const protocol = process.env.ENV === "local" ? req.protocol : "https";return res.status(200).json({diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex 27a926fc8..3274dd93f 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -86,11 +86,12 @@ export async function getMapResults({// If sitemapOnly is true, only get links from sitemapif (crawlerOptions.sitemapOnly) {- const sitemap = await crawler.tryGetSitemap(true, true);- if (sitemap !== null) {- sitemap.forEach((x) => {- links.push(x.url);+ const sitemap = await crawler.tryGetSitemap(urls => {+ urls.forEach((x) => {+ links.push(x);});+ }, true, true);+ if (sitemap > 0) {links = links.slice(1).map((x) => {@@ -143,8 +144,10 @@ export async function getMapResults({}// Parallelize sitemap fetch with serper search- const [sitemap, ...searchResults] = await Promise.all([- ignoreSitemap ? null : crawler.tryGetSitemap(true),+ const [_, ...searchResults] = await Promise.all([+ ignoreSitemap ? null : crawler.tryGetSitemap(urls => {+ links.push(...urls);+ }, true),...(cachedResult ? [] : pagePromises),]);@@ -152,12 +155,6 @@ export async function getMapResults({allResults = searchResults;}- if (sitemap !== null) {- sitemap.forEach((x) => {- links.push(x.url);- });- }-mapResults = allResults.flat().filter((result) => result !== null && result !== undefined);diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.tsindex 41bee2d6d..52bea9e5b 100644--- a/apps/api/src/scraper/WebScraper/crawler.ts+++ b/apps/api/src/scraper/WebScraper/crawler.ts@@ -4,9 +4,10 @@ import { URL } from "url";import { getLinksFromSitemap } from "./sitemap";import robotsParser from "robots-parser";import { getURLDepth } from "./utils/maxDepthUtils";-import { axiosTimeout } from "../../../src/lib/timeout";-import { logger as _logger } from "../../../src/lib/logger";+import { axiosTimeout } from "../../lib/timeout";+import { logger as _logger } from "../../lib/logger";import https from "https";+import { redisConnection } from "../../services/queue-service";export class WebCrawler {private jobId: string;private initialUrl: string;@@ -198,26 +199,60 @@ export class WebCrawler {}public async tryGetSitemap(+ urlsHandler: (urls: string[]) => unknown,fromMap: boolean = false,onlySitemap: boolean = false,- ): Promise<{ url: string; html: string }[] | null> {+ ): Promise<number> {this.logger.debug(`Fetching sitemap links from ${this.initialUrl}`, {method: "tryGetSitemap",});- const sitemapLinks = await this.tryFetchSitemapLinks(this.initialUrl);- if (fromMap && onlySitemap) {- return sitemapLinks.map((link) => ({ url: link, html: "" }));- }- if (sitemapLinks.length > 0) {- let filteredLinks = this.filterLinks(- [...new Set(sitemapLinks)],- this.limit,- this.maxCrawledDepth,- fromMap,- );- return filteredLinks.map((link) => ({ url: link, html: "" }));+ let leftOfLimit = this.limit;++ const normalizeUrl = (url: string) => {+ url = url.replace(/^https?:\/\//, "").replace(/^www\./, "");+ if (url.endsWith("/")) {+ url = url.slice(0, -1);+ }+ return url;+ };++ const _urlsHandler = async (urls: string[]) => {+ let uniqueURLs: string[] = [];+ for (const url of urls) {+ if (await redisConnection.sadd("sitemap:" + this.jobId + ":links", normalizeUrl(url))) {+ uniqueURLs.push(url);+ }+ }++ await redisConnection.expire("sitemap:" + this.jobId + ":links", 3600, "NX");+ if (uniqueURLs.length > 0) {+ urlsHandler(uniqueURLs);+ }+ };++ let count = await this.tryFetchSitemapLinks(this.initialUrl, (urls: string[]) => {+ if (fromMap && onlySitemap) {+ return urlsHandler(urls);+ } else {+ let filteredLinks = this.filterLinks(+ [...new Set(urls)],+ leftOfLimit,+ this.maxCrawledDepth,+ fromMap,+ );+ leftOfLimit -= filteredLinks.length;+ return _urlsHandler(filteredLinks);+ }+ });++ if (count > 0) {+ if (await redisConnection.sadd("sitemap:" + this.jobId + ":links", normalizeUrl(this.initialUrl))) {+ urlsHandler([this.initialUrl]);+ }+ count++;}- return null;++ return count;}public filterURL(href: string, url: string): string | null {@@ -436,22 +471,15 @@ export class WebCrawler {return socialMediaOrEmail.some((ext) => url.includes(ext));}- private async tryFetchSitemapLinks(url: string): Promise<string[]> {- const normalizeUrl = (url: string) => {- url = url.replace(/^https?:\/\//, "").replace(/^www\./, "");- if (url.endsWith("/")) {- url = url.slice(0, -1);- }- return url;- };-+ private async tryFetchSitemapLinks(url: string, urlsHandler: (urls: string[]) => unknown): Promise<number> {const sitemapUrl = url.endsWith(".xml") ? url : `${url}/sitemap.xml`;- let sitemapLinks: string[] = [];++ let sitemapCount: number = 0;// Try to get sitemap from the provided URL firsttry {- sitemapLinks = await getLinksFromSitemap(- { sitemapUrl, allUrls: [], mode: "fire-engine" },+ sitemapCount = await getLinksFromSitemap(+ { sitemapUrl, urlsHandler, mode: "fire-engine" },this.logger,);} catch (error) {@@ -476,20 +504,18 @@ export class WebCrawler {try {// Get all links from the main domain's sitemap- const mainDomainLinks = await getLinksFromSitemap(- { sitemapUrl: mainDomainSitemapUrl, allUrls: [], mode: "fire-engine" },+ sitemapCount += await getLinksFromSitemap(+ { sitemapUrl: mainDomainSitemapUrl, urlsHandler(urls) {+ urlsHandler(urls.filter(link => {+ try {+ const linkUrl = new URL(link);+ return linkUrl.hostname.endsWith(hostname);+ } catch {+ }+ }))+ }, mode: "fire-engine" },this.logger,);- // Filter links to only include those pointing to the current subdomain- const subdomainLinks = mainDomainLinks.filter(link => {- try {- const linkUrl = new URL(link);- return linkUrl.hostname.endsWith(hostname);- } catch {- return false;- }- });- sitemapLinks = [...new Set([...sitemapLinks, ...subdomainLinks])];} catch (error) {this.logger.debug(`Failed to fetch main domain sitemap from ${mainDomainSitemapUrl}`,@@ -506,15 +532,13 @@ export class WebCrawler {}// If no sitemap found yet, try the baseUrl as a last resort- if (sitemapLinks.length === 0) {+ if (sitemapCount === 0) {const baseUrlSitemap = `${this.baseUrl}/sitemap.xml`;try {- const baseLinks = await getLinksFromSitemap(- { sitemapUrl: baseUrlSitemap, allUrls: [], mode: "fire-engine" },+ sitemapCount += await getLinksFromSitemap(+ { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },this.logger,);-- sitemapLinks = [...new Set([...sitemapLinks, ...baseLinks])];} catch (error) {this.logger.debug(`Failed to fetch sitemap from ${baseUrlSitemap}`, {method: "tryFetchSitemapLinks",@@ -524,25 +548,14 @@ export class WebCrawler {if (error instanceof AxiosError && error.response?.status === 404) {// ignore 404} else {- sitemapLinks = await getLinksFromSitemap(- { sitemapUrl: baseUrlSitemap, mode: "fire-engine" },+ sitemapCount += await getLinksFromSitemap(+ { sitemapUrl: baseUrlSitemap, urlsHandler, mode: "fire-engine" },this.logger,);}}}- const normalizedUrl = normalizeUrl(url);- const normalizedSitemapLinks = sitemapLinks.map((link) =>- normalizeUrl(link),- );- // has to be greater than 0 to avoid adding the initial URL to the sitemap links, and preventing crawler to crawl- if (- !normalizedSitemapLinks.includes(normalizedUrl) &&- sitemapLinks.length > 0- ) {- sitemapLinks.push(url);- }- return sitemapLinks;+ return sitemapCount;}}diff --git a/apps/api/src/scraper/WebScraper/sitemap.ts b/apps/api/src/scraper/WebScraper/sitemap.tsindex 2529c022d..8028d225c 100644--- a/apps/api/src/scraper/WebScraper/sitemap.ts+++ b/apps/api/src/scraper/WebScraper/sitemap.ts@@ -11,15 +11,15 @@ const useFireEngine =export async function getLinksFromSitemap({sitemapUrl,- allUrls = [],+ urlsHandler,mode = "axios",}: {sitemapUrl: string;- allUrls?: string[];+ urlsHandler(urls: string[]): unknown,mode?: "axios" | "fire-engine";},logger: Logger,-): Promise<string[]> {+): Promise<number> {try {let content: string = "";try {@@ -31,9 +31,12 @@ export async function getLinksFromSitemap({ forceEngine: "fire-engine;tlsclient", v0DisableJsDom: true },);if (!response.success) {- throw response.error;+ logger.debug("Failed to scrape sitemap via TLSClient, falling back to axios...", { error: response.error })+ const ar = await axios.get(sitemapUrl, { timeout: axiosTimeout });+ content = ar.data;+ } else {+ content = response.document.rawHtml!;}- content = response.document.rawHtml!;} else {const response = await axios.get(sitemapUrl, { timeout: axiosTimeout });content = response.data;@@ -45,11 +48,13 @@ export async function getLinksFromSitemap(sitemapUrl,error,});- return allUrls;++ return 0;}const parsed = await parseStringPromise(content);const root = parsed.urlset || parsed.sitemapindex;+ let count = 0;if (root && root.sitemap) {// Handle sitemap index files@@ -57,20 +62,18 @@ export async function getLinksFromSitemap(.filter((sitemap) => sitemap.loc && sitemap.loc.length > 0).map((sitemap) => sitemap.loc[0]);- const sitemapPromises = sitemapUrls.map((sitemapUrl) =>+ const sitemapPromises: Promise<number>[] = sitemapUrls.map((sitemapUrl) =>getLinksFromSitemap(- { sitemapUrl, allUrls: [], mode },+ { sitemapUrl, urlsHandler, mode },logger,),);const results = await Promise.all(sitemapPromises);- results.forEach(urls => {- allUrls.push(...urls);- });+ count = results.reduce((a,x) => a + x)} else if (root && root.url) {// Check if any URLs point to additional sitemaps- const xmlSitemaps = root.url+ const xmlSitemaps: string[] = root.url.filter((url) =>url.loc &&@@ -83,18 +86,13 @@ export async function getLinksFromSitemap(// Recursively fetch links from additional sitemapsconst sitemapPromises = xmlSitemaps.map((sitemapUrl) =>getLinksFromSitemap(- { sitemapUrl, allUrls: [], mode },+ { sitemapUrl: sitemapUrl, urlsHandler, mode },logger,),);-- const results = await Promise.all(sitemapPromises);- results.forEach(urls => {- allUrls.push(...urls);- });+ count += (await Promise.all(sitemapPromises)).reduce((a,x) => a + x, 0);}- // Add regular URLs that aren't sitemapsconst validUrls = root.url.filter((url) =>@@ -104,8 +102,11 @@ export async function getLinksFromSitemap(!WebCrawler.prototype.isFile(url.loc[0]),).map((url) => url.loc[0]);- allUrls.push(...validUrls);+ count += validUrls.length;+ urlsHandler(validUrls);}++ return count;} catch (error) {logger.debug(`Error processing sitemapUrl: ${sitemapUrl}`, {method: "getLinksFromSitemap",@@ -115,7 +116,7 @@ export async function getLinksFromSitemap(});}- return [...new Set(allUrls)];+ return 0;}export const fetchSitemapData = async (diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 654f6cda0..f59babe4f 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -29,7 +29,7 @@ async function _addScrapeJobToConcurrencyQueue(});}-async function _addScrapeJobToBullMQ(+export async function _addScrapeJobToBullMQ(webScraperOptions: any,options: any,jobId: string,@@ -138,7 +138,6 @@ export async function addScrapeJobs(if (jobs[0].data && jobs[0].data.team_id && jobs[0].data.plan) {const now = Date.now();const limit = await getConcurrencyLimitMax(jobs[0].data.plan);- console.log("CC limit", limit);cleanOldConcurrencyLimitEntries(jobs[0].data.team_id, now);countCanBeDirectlyAdded = Math.max(diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 4ef9610de..e8c8bdf30 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -18,16 +18,18 @@ import { v4 as uuidv4 } from "uuid";import {addCrawlJob,addCrawlJobDone,+ addCrawlJobs,crawlToCrawler,finishCrawl,generateURLPermutations,getCrawl,getCrawlJobs,lockURL,+ lockURLs,normalizeURL,} from "../lib/crawl-redis";import { StoredCrawl } from "../lib/crawl-redis";-import { addScrapeJob } from "./queue-jobs";+import { addScrapeJob, addScrapeJobs } from "./queue-jobs";import {addJobPriority,deleteJobPriority,@@ -191,22 +193,34 @@ const processJobInternal = async (token: string, job: Job & { id: string }) => {await addJobPriority(job.data.team_id, job.id);let err = null;try {- const result = await processJob(job, token);- if (result.success) {- try {- if (job.data.crawl_id && process.env.USE_DB_AUTHENTICATION === "true") {- logger.debug(- "Job succeeded -- has crawl associated, putting null in Redis",- );+ if (job.data?.mode === "kickoff") {+ const result = await processKickoffJob(job, token);+ if (result.success) {+ try {await job.moveToCompleted(null, token, false);- } else {- logger.debug("Job succeeded -- putting result in Redis");- await job.moveToCompleted(result.document, token, false);- }- } catch (e) {}+ } catch (e) {}+ } else {+ logger.debug("Job failed", { result, mode: job.data.mode });+ await job.moveToFailed((result as any).error, token, false);+ }} else {- logger.debug("Job failed", { result });- await job.moveToFailed((result as any).error, token, false);+ const result = await processJob(job, token);+ if (result.success) {+ try {+ if (job.data.crawl_id && process.env.USE_DB_AUTHENTICATION === "true") {+ logger.debug(+ "Job succeeded -- has crawl associated, putting null in Redis",+ );+ await job.moveToCompleted(null, token, false);+ } else {+ logger.debug("Job succeeded -- putting result in Redis");+ await job.moveToCompleted(result.document, token, false);+ }+ } catch (e) {}+ } else {+ logger.debug("Job failed", { result });+ await job.moveToFailed((result as any).error, token, false);+ }}} catch (error) {logger.debug("Job failed", { error });@@ -379,6 +393,130 @@ const workerFun = async (workerFun(getScrapeQueue(), processJobInternal);+async function processKickoffJob(job: Job & { id: string }, token: string) {+ const logger = _logger.child({+ module: "queue-worker",+ method: "processKickoffJob",+ jobId: job.id,+ scrapeId: job.id,+ crawlId: job.data?.crawl_id ?? undefined,+ teamId: job.data?.team_id ?? undefined,+ });++ try {+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;+ const crawler = crawlToCrawler(job.data.crawl_id, sc);++ const sitemap = sc.crawlerOptions.ignoreSitemap+ ? 0+ : await crawler.tryGetSitemap(async urls => {+ if (urls.length === 0) return;++ logger.debug("Using sitemap chunk of length " + urls.length, {+ sitemapLength: urls.length,+ });++ let jobPriority = await getJobPriority({+ plan: job.data.plan,+ team_id: job.data.team_id,+ basePriority: 21,+ });+ logger.debug("Using job priority " + jobPriority, { jobPriority });++ const jobs = urls.map(url => {+ const uuid = uuidv4();+ return {+ name: uuid,+ data: {+ url,+ mode: "single_urls" as const,+ team_id: job.data.team_id,+ plan: job.data.plan!,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: job.data.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: job.data.origin,+ crawl_id: job.data.crawl_id,+ sitemapped: true,+ webhook: job.data.webhook,+ v1: job.data.v1,+ },+ opts: {+ jobId: uuid,+ priority: 20,+ },+ };+ });++ logger.debug("Locking URLs...");+ await lockURLs(+ job.data.crawl_id,+ sc,+ jobs.map((x) => x.data.url),+ );+ logger.debug("Adding scrape jobs to Redis...");+ await addCrawlJobs(+ job.data.crawl_id,+ jobs.map((x) => x.opts.jobId),+ );+ logger.debug("Adding scrape jobs to BullMQ...");+ await addScrapeJobs(jobs);+ });++ if (sitemap === 0) {+ logger.debug("Sitemap not found or ignored.", {+ ignoreSitemap: sc.crawlerOptions.ignoreSitemap,+ });++ logger.debug("Locking URL...");+ await lockURL(job.data.crawl_id, sc, job.data.url);+ const jobId = uuidv4();+ logger.debug("Adding scrape job to Redis...", { jobId });+ await addScrapeJob(+ {+ url: job.data.url,+ mode: "single_urls",+ team_id: job.data.team_id,+ crawlerOptions: job.data.crawlerOptions,+ scrapeOptions: scrapeOptions.parse(job.data.scrapeOptions),+ internalOptions: sc.internalOptions,+ plan: job.data.plan!,+ origin: job.data.origin,+ crawl_id: job.data.crawl_id,+ webhook: job.data.webhook,+ v1: job.data.v1,+ },+ {+ priority: 15,+ },+ jobId,+ );+ logger.debug("Adding scrape job to BullMQ...", { jobId });+ await addCrawlJob(job.data.crawl_id, jobId);+ }+ logger.debug("Done queueing jobs!");++ if (job.data.webhook) {+ logger.debug("Calling webhook with crawl.started...", {+ webhook: job.data.webhook,+ });+ await callWebhook(+ job.data.team_id,+ job.data.crawl_id,+ null,+ job.data.webhook,+ true,+ "crawl.started",+ );+ }++ return { success: true }+ } catch (error) {+ logger.error("An error occurred!", { error })+ return { success: false, error };+ }+}+async function processJob(job: Job & { id: string }, token: string) {const logger = _logger.child({module: "queue-worker",