Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

test: add RequestQueue lruCache test (#1876)

Vlad Franguf649c9dbdae8
diff --git a/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts b/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts
new file mode 100644
index 000000000..0f237d0e0
--- /dev/null
+++ b/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts
@@ -0,0 +1,52 @@
+import { MemoryStorage } from '@crawlee/memory-storage';
+import type { RequestQueueInfo } from '@crawlee/types';
+import { Configuration, RequestQueue } from 'crawlee';
+
+const originalClient = Configuration.getStorageClient();
+Configuration.useStorageClient(new MemoryStorage({ persistStorage: false, writeMetadata: false }));
+
+afterAll(() => {
+ Configuration.useStorageClient(originalClient);
+});
+
+let requestQueueInfo: RequestQueueInfo;
+
+beforeAll(async () => {
+ requestQueueInfo = await Configuration.getStorageClient().requestQueues().getOrCreate('test-request-queue-not-called-on-cached-request');
+});
+
+describe('RequestQueue#addRequest should not call the API if the request is already in the queue', () => {
+ test('should not call the API if the request is already in the queue', async () => {
+ const requestQueue = new RequestQueue({ id: requestQueueInfo.id, client: Configuration.getStorageClient() });
+
+ const clientSpy = jest.spyOn(requestQueue.client, 'addRequest');
+
+ const requestData = await requestQueue.addRequest({ url: 'https://example.com' });
+
+ expect(clientSpy).toHaveBeenCalledTimes(1);
+
+ await requestQueue.markRequestHandled({ id: requestData.requestId, uniqueKey: requestData.uniqueKey } as any);
+
+ await requestQueue.addRequest({ url: 'https://example.com' });
+
+ expect(clientSpy).toHaveBeenCalledTimes(1);
+ });
+});
+
+describe('RequestQueue#addRequests should not call the API if the request is already in the queue', () => {
+ test('should not call the API if the request is already in the queue', async () => {
+ const requestQueue = new RequestQueue({ id: requestQueueInfo.id, client: Configuration.getStorageClient() });
+
+ const clientSpy = jest.spyOn(requestQueue.client, 'batchAddRequests');
+
+ const requestData = await requestQueue.addRequests([{ url: 'https://example2.com' }]);
+
+ expect(clientSpy).toHaveBeenCalledTimes(1);
+
+ await requestQueue.markRequestHandled({ id: requestData.processedRequests[0].requestId, uniqueKey: requestData.processedRequests[0].uniqueKey } as any);
+
+ await requestQueue.addRequests([{ url: 'https://example2.com' }]);
+
+ expect(clientSpy).toHaveBeenCalledTimes(1);
+ });
+});