Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

chore: try to reduce fs calls for listing heads (#2603)

Vlad Frangu4b507228a3e6
diff --git a/packages/memory-storage/src/fs/request-queue/fs.ts b/packages/memory-storage/src/fs/request-queue/fs.ts
index a4937e10e..f5be082f8 100644
--- a/packages/memory-storage/src/fs/request-queue/fs.ts
+++ b/packages/memory-storage/src/fs/request-queue/fs.ts
@@ -22,6 +22,8 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Intern
*/
private sweepTimeout?: NodeJS.Timeout;
+ public orderNo?: number | null;
+
constructor(options: CreateStorageImplementationOptions) {
this.filePath = resolve(options.storeDirectory, `${options.requestId}.json`);
}
@@ -40,6 +42,8 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Intern
const req = JSON.parse(await readFile(this.filePath, 'utf-8'));
this.data = req;
+ this.orderNo = req.orderNo;
+
return req;
});
} finally {
@@ -58,6 +62,8 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Intern
}
await lockAndWrite(this.filePath, data);
+
+ this.orderNo = data.orderNo;
} finally {
this.setOrRefreshSweepTimeout();
this.fsQueue.shift();
diff --git a/packages/memory-storage/src/fs/request-queue/index.ts b/packages/memory-storage/src/fs/request-queue/index.ts
index 7304afe6a..25662a4fb 100644
--- a/packages/memory-storage/src/fs/request-queue/index.ts
+++ b/packages/memory-storage/src/fs/request-queue/index.ts
@@ -1,11 +1,7 @@
import { RequestQueueFileSystemEntry } from './fs';
import { RequestQueueMemoryEntry } from './memory';
-import type { InternalRequest } from '../../resource-clients/request-queue';
-import type { StorageImplementation } from '../common';
-export function createRequestQueueStorageImplementation(
- options: CreateStorageImplementationOptions,
-): StorageImplementation<InternalRequest> {
+export function createRequestQueueStorageImplementation(options: CreateStorageImplementationOptions) {
if (options.persistStorage) {
return new RequestQueueFileSystemEntry(options);
}
diff --git a/packages/memory-storage/src/fs/request-queue/memory.ts b/packages/memory-storage/src/fs/request-queue/memory.ts
index fb4e982bd..79811781b 100644
--- a/packages/memory-storage/src/fs/request-queue/memory.ts
+++ b/packages/memory-storage/src/fs/request-queue/memory.ts
@@ -4,12 +4,15 @@ import type { StorageImplementation } from '../common';
export class RequestQueueMemoryEntry implements StorageImplementation<InternalRequest> {
private data!: InternalRequest;
+ public orderNo?: number | null;
+
async get() {
return this.data;
}
update(data: InternalRequest) {
this.data = data;
+ this.orderNo = data.orderNo;
}
delete() {
diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.ts
index 2191316c7..110395667 100644
--- a/packages/memory-storage/src/resource-clients/request-queue.ts
+++ b/packages/memory-storage/src/resource-clients/request-queue.ts
@@ -6,12 +6,13 @@ import type * as storage from '@crawlee/types';
import { AsyncQueue } from '@sapphire/async-queue';
import { s } from '@sapphire/shapeshift';
import { move } from 'fs-extra';
+import type { RequestQueueFileSystemEntry } from 'packages/memory-storage/src/fs/request-queue/fs';
+import type { RequestQueueMemoryEntry } from 'packages/memory-storage/src/fs/request-queue/memory';
import { BaseClient } from './common/base-client';
import { scheduleBackgroundTask } from '../background-handler';
import { findRequestQueueByPossibleId } from '../cache-helpers';
import { StorageTypes } from '../consts';
-import type { StorageImplementation } from '../fs/common';
import { createRequestQueueStorageImplementation } from '../fs/request-queue';
import type { MemoryStorage } from '../index';
import { purgeNullsFromObject, uniqueKeyToRequestId } from '../utils';
@@ -60,7 +61,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
requestQueueDirectory: string;
private readonly mutex = new AsyncQueue();
- private readonly requests = new Map<string, StorageImplementation<InternalRequest>>();
+ private readonly requests = new Map<string, RequestQueueFileSystemEntry | RequestQueueMemoryEntry>();
private readonly client: MemoryStorage;
constructor(options: RequestQueueClientOptions) {
@@ -173,10 +174,19 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
break;
}
- const request = await storageEntry.get();
+ let { orderNo } = storageEntry;
+ let loaded: InternalRequest;
- if (request.orderNo) {
- items.push(request);
+ // Uncached entry
+ if (typeof orderNo === 'undefined') {
+ loaded = await storageEntry.get();
+
+ orderNo = loaded.orderNo;
+ }
+
+ // Have an order no -> fetch from fs/memory and return
+ if (orderNo) {
+ items.push(await storageEntry.get());
}
}
@@ -212,6 +222,11 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
break;
}
+ // This is set to null when the request has been handled, so we don't need to re-fetch from fs
+ if (storageEntry.orderNo === null) {
+ continue;
+ }
+
// Always fetch from fs, as this also locks and we do not want to end up in a state where another process locked the request but we have cached it as unlocked
const request = await storageEntry.get(true);