Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
perf: optimize `RequestList` memory footprint (#2466)
Martin Adámek12210bd191b5
diff --git a/packages/core/src/request.ts b/packages/core/src/request.tsindex 6718c6bae..643143590 100644--- a/packages/core/src/request.ts+++ b/packages/core/src/request.ts@@ -193,7 +193,7 @@ export class Request<UserData extends Dictionary = Dictionary> {this.id = id;this.url = url;this.loadedUrl = loadedUrl;- this.uniqueKey = uniqueKey || this._computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });+ this.uniqueKey = uniqueKey || Request.computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey });this.method = method;this.payload = payload;this.noRetry = noRetry;@@ -378,7 +378,18 @@ export class Request<UserData extends Dictionary = Dictionary> {this.errorMessages.push(message);}- protected _computeUniqueKey({ url, method, payload, keepUrlFragment, useExtendedUniqueKey }: ComputeUniqueKeyOptions) {+ // TODO: only for better BC, remove in v4+ protected _computeUniqueKey(options: ComputeUniqueKeyOptions) {+ return Request.computeUniqueKey(options);+ }++ // TODO: only for better BC, remove in v4+ protected _hashPayload(payload: BinaryLike): string {+ return Request.hashPayload(payload);+ }++ /** @internal */+ static computeUniqueKey({ url, method = 'GET', payload, keepUrlFragment = false, useExtendedUniqueKey = false }: ComputeUniqueKeyOptions) {const normalizedMethod = method.toUpperCase();const normalizedUrl = normalizeUrl(url, keepUrlFragment) || url; // It returns null when url is invalid, causing weird errors.if (!useExtendedUniqueKey) {@@ -390,11 +401,12 @@ export class Request<UserData extends Dictionary = Dictionary> {}return normalizedUrl;}- const payloadHash = payload ? this._hashPayload(payload) : '';+ const payloadHash = payload ? Request.hashPayload(payload) : '';return `${normalizedMethod}(${payloadHash}):${normalizedUrl}`;}- protected _hashPayload(payload: BinaryLike): string {+ /** @internal */+ static hashPayload(payload: BinaryLike): string {return crypto.createHash('sha256').update(payload)diff --git a/packages/core/src/storages/request_list.ts b/packages/core/src/storages/request_list.tsindex 9eeb4f80e..6c2d6b0e9 100644--- a/packages/core/src/storages/request_list.ts+++ b/packages/core/src/storages/request_list.ts@@ -9,8 +9,7 @@ import type { EventManager } from '../events';import { EventType } from '../events';import { log } from '../log';import type { ProxyConfiguration } from '../proxy_configuration';-import type { InternalSource, RequestOptions, Source } from '../request';-import { Request } from '../request';+import { type InternalSource, type RequestOptions, Request, type Source } from '../request';import { createDeserialize, serializeArray } from '../serialization';/** @internal */@@ -238,7 +237,7 @@ export class RequestList {* All requests in the array have distinct uniqueKey!* @internal*/- requests: Request[] = [];+ requests: (Request | RequestOptions)[] = [];/** Index to the next item in requests array to fetch. All previous requests are either handled or in progress. */private nextIndex = 0;@@ -551,7 +550,7 @@ export class RequestList {return {nextIndex: this.nextIndex,nextUniqueKey: this.nextIndex < this.requests.length- ? this.requests[this.nextIndex].uniqueKey+ ? this.requests[this.nextIndex].uniqueKey!: null,inProgress: [...this.inProgress],};@@ -593,21 +592,31 @@ export class RequestList {if (uniqueKey) {this.reclaimed.delete(uniqueKey);const index = this.uniqueKeyToIndex[uniqueKey];- return this.requests[index];+ return this.ensureRequest(this.requests[index], index);}// Otherwise return next request.if (this.nextIndex < this.requests.length) {- const request = this.requests[this.nextIndex];- this.inProgress.add(request.uniqueKey);+ const index = this.nextIndex;+ const request = this.requests[index];+ this.inProgress.add(request.uniqueKey!);this.nextIndex++;this.isStatePersisted = false;- return request;+ return this.ensureRequest(request, index);}return null;}+ private ensureRequest(requestLike: Request | RequestOptions, index: number): Request {+ if (requestLike instanceof Request) {+ return requestLike;+ }++ this.requests[index] = new Request(requestLike);+ return this.requests[index] as Request;+ }+/*** Marks request as handled after successful processing.*/@@ -694,19 +703,21 @@ export class RequestList {* of a `Request`, then the function creates a `Request` instance.*/protected _addRequest(source: RequestListSource) {- let request;+ let request: Request | RequestOptions;const type = typeof source;+if (type === 'string') {- request = new Request({ url: source as string });+ request = { url: source as string };} else if (source instanceof Request) {request = source;} else if (source && type === 'object') {- request = new Request(source as RequestOptions);+ request = source as RequestOptions;} else {throw new Error(`Cannot create Request from type: ${type}`);}const hasUniqueKey = Reflect.has(Object(source), 'uniqueKey');+ request.uniqueKey ??= Request.computeUniqueKey(request as any);// Add index to uniqueKey if duplicates are to be keptif (this.keepDuplicateUrls && !hasUniqueKey) {diff --git a/test/core/request_list.test.ts b/test/core/request_list.test.tsindex 539695b88..2f60e6613 100644--- a/test/core/request_list.test.ts+++ b/test/core/request_list.test.ts@@ -728,7 +728,7 @@ describe('RequestList', () => {const name = 'xxx';const SDK_KEY = `SDK_${name}`;const sources = ['https://example.com'];- const requests = sources.map((url) => new Request({ url }));+ const requests = sources.map((url) => ({ url, uniqueKey: url }));const rl = await RequestList.open(name, sources);expect(rl).toBeInstanceOf(RequestList);@@ -752,7 +752,7 @@ describe('RequestList', () => {const SDK_KEY = `SDK_${name}`;let counter = 0;const sources = [{ url: 'https://example.com' }];- const requests = sources.map(({ url }) => new Request({ url, uniqueKey: `${url}-${counter++}` }));+ const requests = sources.map(({ url }) => ({ url, uniqueKey: `${url}-${counter++}` }));const options = {keepDuplicateUrls: true,persistStateKey: 'yyy',@@ -780,7 +780,7 @@ describe('RequestList', () => {const name: string = null;const sources = [{ url: 'https://example.com' }];- const requests = sources.map(({ url }) => new Request({ url }));+ const requests = sources.map(({ url }) => ({ url, uniqueKey: url }));const rl = await RequestList.open(name, sources);expect(rl).toBeInstanceOf(RequestList);