Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker): only do redirect handling logic on crawls, not batch scrape

Gergő Móriczb92a4eb79b04
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 19ce3ba09..21c9745cc 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -61,7 +61,7 @@ export async function batchScrapeController(
}
logger.debug("Batch scrape " + id + " starting", {
- urlsLength: urls,
+ urlsLength: urls.length,
appendToId: req.body.appendToId,
account: req.account,
});
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 9e6f3d24e..a48c798b4 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -710,7 +710,8 @@ async function processJob(job: Job & { id: string }, token: string) {
doc.metadata.url !== undefined &&
doc.metadata.sourceURL !== undefined &&
normalizeURL(doc.metadata.url, sc) !==
- normalizeURL(doc.metadata.sourceURL, sc)
+ normalizeURL(doc.metadata.sourceURL, sc) &&
+ job.data.crawlerOptions !== null // only on crawls, don't care on batch scrape
) {
const crawler = crawlToCrawler(job.data.crawl_id, sc);
if (