Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
chore: try to reduce fs calls for listing heads (#2603)
Vlad Frangu4b507228a3e6
diff --git a/packages/memory-storage/src/fs/request-queue/fs.ts b/packages/memory-storage/src/fs/request-queue/fs.tsindex a4937e10e..f5be082f8 100644--- a/packages/memory-storage/src/fs/request-queue/fs.ts+++ b/packages/memory-storage/src/fs/request-queue/fs.ts@@ -22,6 +22,8 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Intern*/private sweepTimeout?: NodeJS.Timeout;+ public orderNo?: number | null;+constructor(options: CreateStorageImplementationOptions) {this.filePath = resolve(options.storeDirectory, `${options.requestId}.json`);}@@ -40,6 +42,8 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Internconst req = JSON.parse(await readFile(this.filePath, 'utf-8'));this.data = req;+ this.orderNo = req.orderNo;+return req;});} finally {@@ -58,6 +62,8 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Intern}await lockAndWrite(this.filePath, data);++ this.orderNo = data.orderNo;} finally {this.setOrRefreshSweepTimeout();this.fsQueue.shift();diff --git a/packages/memory-storage/src/fs/request-queue/index.ts b/packages/memory-storage/src/fs/request-queue/index.tsindex 7304afe6a..25662a4fb 100644--- a/packages/memory-storage/src/fs/request-queue/index.ts+++ b/packages/memory-storage/src/fs/request-queue/index.ts@@ -1,11 +1,7 @@import { RequestQueueFileSystemEntry } from './fs';import { RequestQueueMemoryEntry } from './memory';-import type { InternalRequest } from '../../resource-clients/request-queue';-import type { StorageImplementation } from '../common';-export function createRequestQueueStorageImplementation(- options: CreateStorageImplementationOptions,-): StorageImplementation<InternalRequest> {+export function createRequestQueueStorageImplementation(options: CreateStorageImplementationOptions) {if (options.persistStorage) {return new RequestQueueFileSystemEntry(options);}diff --git a/packages/memory-storage/src/fs/request-queue/memory.ts b/packages/memory-storage/src/fs/request-queue/memory.tsindex fb4e982bd..79811781b 100644--- a/packages/memory-storage/src/fs/request-queue/memory.ts+++ b/packages/memory-storage/src/fs/request-queue/memory.ts@@ -4,12 +4,15 @@ import type { StorageImplementation } from '../common';export class RequestQueueMemoryEntry implements StorageImplementation<InternalRequest> {private data!: InternalRequest;+ public orderNo?: number | null;+async get() {return this.data;}update(data: InternalRequest) {this.data = data;+ this.orderNo = data.orderNo;}delete() {diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.tsindex 2191316c7..110395667 100644--- a/packages/memory-storage/src/resource-clients/request-queue.ts+++ b/packages/memory-storage/src/resource-clients/request-queue.ts@@ -6,12 +6,13 @@ import type * as storage from '@crawlee/types';import { AsyncQueue } from '@sapphire/async-queue';import { s } from '@sapphire/shapeshift';import { move } from 'fs-extra';+import type { RequestQueueFileSystemEntry } from 'packages/memory-storage/src/fs/request-queue/fs';+import type { RequestQueueMemoryEntry } from 'packages/memory-storage/src/fs/request-queue/memory';import { BaseClient } from './common/base-client';import { scheduleBackgroundTask } from '../background-handler';import { findRequestQueueByPossibleId } from '../cache-helpers';import { StorageTypes } from '../consts';-import type { StorageImplementation } from '../fs/common';import { createRequestQueueStorageImplementation } from '../fs/request-queue';import type { MemoryStorage } from '../index';import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils';@@ -60,7 +61,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQuerequestQueueDirectory: string;private readonly mutex = new AsyncQueue();- private readonly requests = new Map<string, StorageImplementation<InternalRequest>>();+ private readonly requests = new Map<string, RequestQueueFileSystemEntry | RequestQueueMemoryEntry>();private readonly client: MemoryStorage;constructor(options: RequestQueueClientOptions) {@@ -173,10 +174,19 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQuebreak;}- const request = await storageEntry.get();+ let { orderNo } = storageEntry;+ let loaded: InternalRequest;- if (request.orderNo) {- items.push(request);+ // Uncached entry+ if (typeof orderNo === 'undefined') {+ loaded = await storageEntry.get();++ orderNo = loaded.orderNo;+ }++ // Have an order no -> fetch from fs/memory and return+ if (orderNo) {+ items.push(await storageEntry.get());}}@@ -212,6 +222,11 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQuebreak;}+ // This is set to null when the request has been handled, so we don't need to re-fetch from fs+ if (storageEntry.orderNo === null) {+ continue;+ }+// Always fetch from fs, as this also locks and we do not want to end up in a state where another process locked the request but we have cached it as unlockedconst request = await storageEntry.get(true);