Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
test: add RequestQueue lruCache test (#1876)
Vlad Franguf649c9dbdae8
diff --git a/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts b/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.tsnew file mode 100644index 000000000..0f237d0e0--- /dev/null+++ b/packages/core/test/request-queue/adding-the-same-request-should-not-call-the-api.test.ts@@ -0,0 +1,52 @@+import { MemoryStorage } from '@crawlee/memory-storage';+import type { RequestQueueInfo } from '@crawlee/types';+import { Configuration, RequestQueue } from 'crawlee';++const originalClient = Configuration.getStorageClient();+Configuration.useStorageClient(new MemoryStorage({ persistStorage: false, writeMetadata: false }));++afterAll(() => {+ Configuration.useStorageClient(originalClient);+});++let requestQueueInfo: RequestQueueInfo;++beforeAll(async () => {+ requestQueueInfo = await Configuration.getStorageClient().requestQueues().getOrCreate('test-request-queue-not-called-on-cached-request');+});++describe('RequestQueue#addRequest should not call the API if the request is already in the queue', () => {+ test('should not call the API if the request is already in the queue', async () => {+ const requestQueue = new RequestQueue({ id: requestQueueInfo.id, client: Configuration.getStorageClient() });++ const clientSpy = jest.spyOn(requestQueue.client, 'addRequest');++ const requestData = await requestQueue.addRequest({ url: 'https://example.com' });++ expect(clientSpy).toHaveBeenCalledTimes(1);++ await requestQueue.markRequestHandled({ id: requestData.requestId, uniqueKey: requestData.uniqueKey } as any);++ await requestQueue.addRequest({ url: 'https://example.com' });++ expect(clientSpy).toHaveBeenCalledTimes(1);+ });+});++describe('RequestQueue#addRequests should not call the API if the request is already in the queue', () => {+ test('should not call the API if the request is already in the queue', async () => {+ const requestQueue = new RequestQueue({ id: requestQueueInfo.id, client: Configuration.getStorageClient() });++ const clientSpy = jest.spyOn(requestQueue.client, 'batchAddRequests');++ const requestData = await requestQueue.addRequests([{ url: 'https://example2.com' }]);++ expect(clientSpy).toHaveBeenCalledTimes(1);++ await requestQueue.markRequestHandled({ id: requestData.processedRequests[0].requestId, uniqueKey: requestData.processedRequests[0].uniqueKey } as any);++ await requestQueue.addRequests([{ url: 'https://example2.com' }]);++ expect(clientSpy).toHaveBeenCalledTimes(1);+ });+});