Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker): only do redirect handling logic on crawls, not batch scrape
Gergő Móriczb92a4eb79b04
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 19ce3ba09..21c9745cc 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -61,7 +61,7 @@ export async function batchScrapeController(}logger.debug("Batch scrape " + id + " starting", {- urlsLength: urls,+ urlsLength: urls.length,appendToId: req.body.appendToId,account: req.account,});diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 9e6f3d24e..a48c798b4 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -710,7 +710,8 @@ async function processJob(job: Job & { id: string }, token: string) {doc.metadata.url !== undefined &&doc.metadata.sourceURL !== undefined &&normalizeURL(doc.metadata.url, sc) !==- normalizeURL(doc.metadata.sourceURL, sc)+ normalizeURL(doc.metadata.sourceURL, sc) &&+ job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape) {const crawler = crawlToCrawler(job.data.crawl_id, sc);if (