Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: small code fixes found by the v4 upgrade skill audit (#4361)
Vlad Frangu74ab27fcab78
diff --git a/docs/public-api/crawlee-basic.api.md b/docs/public-api/crawlee-basic.api.mdindex c4629dafb..0d942f492 100644--- a/docs/public-api/crawlee-basic.api.md+++ b/docs/public-api/crawlee-basic.api.md@@ -1014,7 +1014,7 @@ export interface TaskLoopPredicates {export class ThrottlingRequestManager<T extends IRequestManager = IRequestManager> implements IRequestManager {// (undocumented)[Symbol.asyncIterator](): AsyncGenerator<Request_2<Dictionary>, void, unknown>;- constructor(options: ThrottlingRequestManagerOptions<T>, config?: Configuration);+ constructor(options: ThrottlingRequestManagerOptions<T>, configuration?: Configuration);// (undocumented)addRequest(requestLike: Source, options?: RequestQueueOperationOptions): Promise<RequestQueueOperationInfo>;addRequestsBatched(requests: RequestsLike, options?: AddRequestsBatchedOptions): Promise<AddRequestsBatchedResult>;diff --git a/packages/basic-crawler/src/internals/crawlers/statistics.ts b/packages/basic-crawler/src/internals/crawlers/statistics.tsindex 92cdce01a..335651949 100644--- a/packages/basic-crawler/src/internals/crawlers/statistics.ts+++ b/packages/basic-crawler/src/internals/crawlers/statistics.ts@@ -46,7 +46,7 @@ const errorTrackerConfig = {* order here *is* the record's field order (guarded by a test).** JSON has no infinity, so the three fields that are `Infinity` until the first request settles are written as- * `null`. Both {@apilink Statistics.serializeState} and {@apilink Statistics.deserializeState} run through this,+ * `null`. Both the private serialize and deserialize steps of `Statistics` run through this,* which is what keeps them describing the same record.** Nothing is optional on purpose: the record has always carried every field, so one missing a field is not one ofdiff --git a/packages/basic-crawler/src/internals/throttling-request-manager.ts b/packages/basic-crawler/src/internals/throttling-request-manager.tsindex ab13228eb..4f5e05647 100644--- a/packages/basic-crawler/src/internals/throttling-request-manager.ts+++ b/packages/basic-crawler/src/internals/throttling-request-manager.ts@@ -291,7 +291,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManagereadonly #throttleBy: 'hostname' | 'registrableDomain';readonly #maxThrottledDomains: number;readonly #persistStateKey: string;- readonly #config: Configuration;+ readonly #configuration: Configuration;readonly #subManagers = new Map<string, Promise<T>>();@@ -343,15 +343,15 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManageconstructor(options: ThrottlingRequestManagerOptions<T>,- config: Configuration = serviceLocator.getConfiguration(),+ configuration: Configuration = serviceLocator.getConfiguration(),) {- this.#config = config;+ this.#configuration = configuration;parseArgument(options, throttlingRequestManagerOptionsSchema, 'ThrottlingRequestManagerOptions');if (options.inner === undefined) {// Deferred like any other factory, so a manager nobody fetches from opens no storage. The opener is// assigned below and read only when this runs.- this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#config });+ this.#innerFactory = () => this.#requestManagerOpener(null, { configuration: this.#configuration });} else if (typeof options.inner === 'function') {this.#innerFactory = options.inner;} else {@@ -537,7 +537,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManage// Backends use the alias as a directory name, and an IPv6 literal is full of characters// Windows will not accept. Ordinary hostnames survive this untouched.{ alias: `throttled-${encodeURIComponent(domain)}` },- { configuration: this.#config },+ { configuration: this.#configuration },);this.#subManagers.set(domain, subManager);this.#ensureDomainState(domain);@@ -577,7 +577,7 @@ export class ThrottlingRequestManager<T extends IRequestManager = IRequestManageasync #ensureSubManagers(): Promise<void> {this.#subManagersReady ??= (async () => {if (this.#throttlesEveryDomain) {- this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#config });+ this.#domainListStore = await KeyValueStore.open(null, { configuration: this.#configuration });for (const domain of (await this.#domainListStore.getValue<string[]>(this.#persistStateKey)) ?? []) {this.#discoveredDomains.add(domain);diff --git a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts b/packages/browser-pool/src/puppeteer/puppeteer-plugin.tsindex e317aa2d9..f162de5c4 100644--- a/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts+++ b/packages/browser-pool/src/puppeteer/puppeteer-plugin.ts@@ -28,11 +28,7 @@ export class PuppeteerPlugin extends BrowserPlugin<> {protected async _launch(launchContext: PuppeteerLaunchContext): Promise<PuppeteerTypes.Browser> {const oldPuppeteerVersion = await this.isOldPuppeteerVersion();- const { launchOptions, userDataDir, experimentalContainers, proxyUrl } = launchContext;-- if (experimentalContainers) {- throw new Error('Experimental containers are only available with Playwright');- }+ const { launchOptions, userDataDir, proxyUrl } = launchContext;launchOptions!.userDataDir = launchOptions!.userDataDir ?? userDataDir;diff --git a/packages/http-crawler/src/internals/file-download.ts b/packages/http-crawler/src/internals/file-download.tsindex 4a2568ee9..8b9f07729 100644--- a/packages/http-crawler/src/internals/file-download.ts+++ b/packages/http-crawler/src/internals/file-download.ts@@ -45,7 +45,7 @@ export type FileDownloadRequestHandler<* However, it doesn't parse the content - if you need to e.g. extract data from the downloaded files,* you might need to use {@apilink CheerioCrawler}, {@apilink PuppeteerCrawler} or {@apilink PlaywrightCrawler} instead.*- * `FileCrawler` downloads each URL using a plain HTTP request and then invokes the user-provided {@apilink BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.+ * `FileDownload` downloads each URL using a plain HTTP request and then invokes the user-provided {@apilink BasicCrawlerOptions.requestHandler} where the user can specify what to do with the downloaded data.** The source URLs are represented using {@apilink Request} objects that are fed from the {@apilink IRequestManager|request manager} provided via the {@apilink BasicCrawlerOptions.requestManager|`requestManager`} constructor option (a {@apilink RequestQueue} is itself a request manager). To read from a read-only source such as a {@apilink RequestList} while still being able to enqueue new requests, combine it with a queue into a {@apilink RequestManagerTandem} via {@apilink IRequestLoader.toTandem|`requestLoader.toTandem()`} and pass the result as `requestManager`.*@@ -53,14 +53,14 @@ export type FileDownloadRequestHandler<** The crawler finishes when there are no more {@apilink Request} objects to crawl.*- * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@apilink ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileCrawler` constructor, or, for finer control, by injecting a pre-configured {@apilink ConcurrencySystem|`concurrencySystem`}.+ * New requests are only dispatched when there is enough free CPU and memory available, as judged by the crawler's {@apilink ConcurrencySystem}. Concurrency is tuned via the `minConcurrency`, `maxConcurrency` and `maxRequestsPerMinute` options of the `FileDownload` constructor, or, for finer control, by injecting a pre-configured {@apilink ConcurrencySystem|`concurrencySystem`}.** ## Example usage** ```ts- * const crawler = new FileDownloader({- * requestHandler({ body, request }) {- * writeFileSync(request.url.replace(/[^a-z0-9\.]/gi, '_'), body);+ * const crawler = new FileDownload({+ * async requestHandler({ response, request }) {+ * await writeFile(request.url.replace(/[^a-z0-9\.]/gi, '_'), Buffer.from(await response.arrayBuffer()));* },* });*