Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix!: Remove mostly unused Request.userData.forefront field (#4157)

Jan Buchar761be275c6ce
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.ts
index 3960b3b50..4f15f074b 100644
--- a/packages/basic-crawler/src/internals/basic-crawler.ts
+++ b/packages/basic-crawler/src/internals/basic-crawler.ts
@@ -2830,13 +2830,13 @@ export class BasicCrawler<
source: IRequestManager,
): Promise<void> {
if (error instanceof RequestThrottledError) {
- // The domain told us to come back later, so the request was never really attempted. Put it back
- // without recording a failure - it costs neither a retry nor session reputation.
+ // The domain told us to come back later, so the request was never really attempted. Put it back where
+ // it was, without recording a failure - it costs neither a retry nor session reputation.
this.log.debug(`Deferring request because its domain is rate-limiting us. ${error.message}`, {
id: request.id,
url: request.url,
});
- await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+ await source.reclaimRequest(request, { forefront: true });
return;
}
@@ -2873,7 +2873,7 @@ export class BasicCrawler<
retryCount,
});
- await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
+ await source.reclaimRequest(request);
return;
}
}