Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(queue-worker): always crawl links from content even if sitemapped
Gergő Móricz3c045c43a446
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 37e14bafe..672f8b76d 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -300,49 +300,41 @@ async function processJob(job: Job, token: string) {const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;- if (!job.data.sitemapped) {- if (!sc.cancelled) {- const crawler = crawlToCrawler(job.data.crawl_id, sc);-- const links = crawler.filterLinks(- crawler.extractLinksFromHTML(rawHtml ?? "", sc.originUrl),- Infinity,- sc.crawlerOptions?.maxDepth ?? 10- );+ if (!sc.cancelled) {+ const crawler = crawlToCrawler(job.data.crawl_id, sc);++ const links = crawler.filterLinks(+ crawler.extractLinksFromHTML(rawHtml ?? "", sc.originUrl),+ Infinity,+ sc.crawlerOptions?.maxDepth ?? 10+ );- for (const link of links) {- if (await lockURL(job.data.crawl_id, sc, link)) {- // This seems to work really welel- const jobPriority = await getJobPriority({- plan: sc.plan as PlanType,+ for (const link of links) {+ if (await lockURL(job.data.crawl_id, sc, link)) {+ const jobPriority = await getJobPriority({+ plan: sc.plan as PlanType,+ team_id: sc.team_id,+ basePriority: job.data.crawl_id ? 20 : 10,+ });+ const jobId = uuidv4();++ const newJob = await addScrapeJob(+ {+ url: link,+ mode: "single_urls",+ crawlerOptions: sc.crawlerOptions,team_id: sc.team_id,- basePriority: job.data.crawl_id ? 20 : 10,- });- const jobId = uuidv4();-- // console.log("plan: ", sc.plan);- // console.log("team_id: ", sc.team_id)- // console.log("base priority: ", job.data.crawl_id ? 20 : 10)- // console.log("job priority: " , jobPriority, "\n\n\n")-- const newJob = await addScrapeJob(- {- url: link,- mode: "single_urls",- crawlerOptions: sc.crawlerOptions,- team_id: sc.team_id,- pageOptions: sc.pageOptions,- origin: job.data.origin,- crawl_id: job.data.crawl_id,- v1: job.data.v1,- },- {},- jobId,- jobPriority- );+ pageOptions: sc.pageOptions,+ origin: job.data.origin,+ crawl_id: job.data.crawl_id,+ v1: job.data.v1,+ },+ {},+ jobId,+ jobPriority+ );- await addCrawlJob(job.data.crawl_id, newJob.id);- }+ await addCrawlJob(job.data.crawl_id, newJob.id);}}}