Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(MemoryStorage): cache requests in `RequestQueue` (#1899)

Vlad Frangu063dcd1c9e66
diff --git a/packages/memory-storage/src/fs/request-queue/fs.ts b/packages/memory-storage/src/fs/request-queue/fs.ts
index 668d3350f..88cd8bca1 100644
--- a/packages/memory-storage/src/fs/request-queue/fs.ts
+++ b/packages/memory-storage/src/fs/request-queue/fs.ts
@@ -10,6 +10,13 @@ import type { StorageImplementation } from '../common';
export class RequestQueueFileSystemEntry implements StorageImplementation<InternalRequest> {
private filePath: string;
private fsQueue = new AsyncQueue();
+ private data?: InternalRequest;
+
+ /**
+ * A "sweep" timeout that is created/refreshed whenever this entry is accessed/updated.
+ * It exists to ensure that the entry is not kept in memory indefinitely, by sweeping it after 60 seconds of inactivity (in order to keep memory usage low)
+ */
+ private sweepTimeout?: NodeJS.Timeout;
constructor(options: CreateStorageImplementationOptions) {
this.filePath = resolve(options.storeDirectory, `${options.requestId}.json`);
@@ -17,8 +24,18 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Intern
async get() {
await this.fsQueue.wait();
+ this.setOrRefreshSweepTimeout();
+
+ if (this.data) {
+ this.fsQueue.shift();
+ return this.data;
+ }
+
try {
- return JSON.parse(await readFile(this.filePath, 'utf-8'));
+ const req = JSON.parse(await readFile(this.filePath, 'utf-8'));
+ this.data = req;
+
+ return req;
} finally {
this.fsQueue.shift();
}
@@ -26,10 +43,13 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Intern
async update(data: InternalRequest) {
await this.fsQueue.wait();
+ this.data = data;
+
try {
await ensureDir(dirname(this.filePath));
await lockAndWrite(this.filePath, data);
} finally {
+ this.setOrRefreshSweepTimeout();
this.fsQueue.shift();
}
}
@@ -39,4 +59,15 @@ export class RequestQueueFileSystemEntry implements StorageImplementation<Intern
await rm(this.filePath, { force: true });
this.fsQueue.shift();
}
+
+ private setOrRefreshSweepTimeout() {
+ if (this.sweepTimeout) {
+ this.sweepTimeout.refresh();
+ } else {
+ this.sweepTimeout = setTimeout(() => {
+ this.sweepTimeout = undefined;
+ this.data = undefined;
+ }, 60_000).unref();
+ }
+ }
}
diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.ts
index 1a8bbc907..895fd4c36 100644
--- a/packages/memory-storage/src/resource-clients/request-queue.ts
+++ b/packages/memory-storage/src/resource-clients/request-queue.ts
@@ -230,7 +230,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
}).parse(options);
const queue = await this.getQueue();
- const request = await queue.requests.get(id);
+ const request = queue.requests.get(id);
const internalRequest = await request?.get();
@@ -262,7 +262,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQue
}).parse(options);
const queue = await this.getQueue();
- const request = await queue.requests.get(id);
+ const request = queue.requests.get(id);
const internalRequest = await request?.get();