Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "feat(queue-worker): always crawl links from content even if sitemapped"

Nicolas258c67ce6739
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 672f8b76d..37e14bafe 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -300,41 +300,49 @@ async function processJob(job: Job, token: string) {
const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
- if (!sc.cancelled) {
- const crawler = crawlToCrawler(job.data.crawl_id, sc);
-
- const links = crawler.filterLinks(
- crawler.extractLinksFromHTML(rawHtml ?? "", sc.originUrl),
- Infinity,
- sc.crawlerOptions?.maxDepth ?? 10
- );
-
- for (const link of links) {
- if (await lockURL(job.data.crawl_id, sc, link)) {
- const jobPriority = await getJobPriority({
- plan: sc.plan as PlanType,
- team_id: sc.team_id,
- basePriority: job.data.crawl_id ? 20 : 10,
- });
- const jobId = uuidv4();
+ if (!job.data.sitemapped) {
+ if (!sc.cancelled) {
+ const crawler = crawlToCrawler(job.data.crawl_id, sc);
+
+ const links = crawler.filterLinks(
+ crawler.extractLinksFromHTML(rawHtml ?? "", sc.originUrl),
+ Infinity,
+ sc.crawlerOptions?.maxDepth ?? 10
+ );
- const newJob = await addScrapeJob(
- {
- url: link,
- mode: "single_urls",
- crawlerOptions: sc.crawlerOptions,
+ for (const link of links) {
+ if (await lockURL(job.data.crawl_id, sc, link)) {
+ // This seems to work really welel
+ const jobPriority = await getJobPriority({
+ plan: sc.plan as PlanType,
team_id: sc.team_id,
- pageOptions: sc.pageOptions,
- origin: job.data.origin,
- crawl_id: job.data.crawl_id,
- v1: job.data.v1,
- },
- {},
- jobId,
- jobPriority
- );
+ basePriority: job.data.crawl_id ? 20 : 10,
+ });
+ const jobId = uuidv4();
+
+ // console.log("plan: ", sc.plan);
+ // console.log("team_id: ", sc.team_id)
+ // console.log("base priority: ", job.data.crawl_id ? 20 : 10)
+ // console.log("job priority: " , jobPriority, "\n\n\n")
+
+ const newJob = await addScrapeJob(
+ {
+ url: link,
+ mode: "single_urls",
+ crawlerOptions: sc.crawlerOptions,
+ team_id: sc.team_id,
+ pageOptions: sc.pageOptions,
+ origin: job.data.origin,
+ crawl_id: job.data.crawl_id,
+ v1: job.data.v1,
+ },
+ {},
+ jobId,
+ jobPriority
+ );
- await addCrawlJob(job.data.crawl_id, newJob.id);
+ await addCrawlJob(job.data.crawl_id, newJob.id);
+ }
}
}
}