Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: respect current config when creating implicit `RequestQueue` instance
Martin Adámek845141d921c1
diff --git a/packages/basic-crawler/src/internals/basic-crawler.ts b/packages/basic-crawler/src/internals/basic-crawler.tsindex afc1f7ef7..3ca127e19 100644--- a/packages/basic-crawler/src/internals/basic-crawler.ts+++ b/packages/basic-crawler/src/internals/basic-crawler.ts@@ -768,7 +768,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext// ignored - as a failed requests is still handled.if (this.running === false && this.requestQueue?.name === 'default' && purgeRequestQueue) {await this.requestQueue.drop();- this.requestQueue = await RequestQueue.open();+ this.requestQueue = await RequestQueue.open(null, { config: this.config });}this.running = true;@@ -850,7 +850,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext}async getRequestQueue() {- this.requestQueue ??= await RequestQueue.open();+ this.requestQueue ??= await RequestQueue.open(null, { config: this.config });return this.requestQueue!;}