Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(MemoryStorage): RequestQueue should respect `forefront` (#1816)
Vlad Frangub68e86a97954
diff --git a/packages/memory-storage/src/resource-clients/request-queue.ts b/packages/memory-storage/src/resource-clients/request-queue.tsindex 243b50d20..eb2dbaf79 100644--- a/packages/memory-storage/src/resource-clients/request-queue.ts+++ b/packages/memory-storage/src/resource-clients/request-queue.ts@@ -159,7 +159,7 @@ export class RequestQueueClient extends BaseClient implements storage.RequestQuelimit,hadMultipleClients: false,queueModifiedAt: existingQueueById.modifiedAt,- items: items.map(({ json }) => this._jsonToRequest(json)!),+ items: items.sort((a, b) => a.orderNo! - b.orderNo!).map(({ json }) => this._jsonToRequest(json)!),};}diff --git a/packages/memory-storage/test/request-queue/forefront.test.ts b/packages/memory-storage/test/request-queue/forefront.test.tsnew file mode 100644index 000000000..33b042507--- /dev/null+++ b/packages/memory-storage/test/request-queue/forefront.test.ts@@ -0,0 +1,59 @@+import { MemoryStorage } from '@crawlee/memory-storage';+import type { RequestQueueClient } from '@crawlee/types';+import { setTimeout as sleep } from 'node:timers/promises';++describe('RequestQueue forefront should be respected when listing head', () => {+ const storage = new MemoryStorage({+ persistStorage: false,+ });++ let requestQueue: RequestQueueClient;++ beforeEach(async () => {+ const { id } = await storage.requestQueues().getOrCreate('forefront');+ requestQueue = storage.requestQueue(id);+ });++ afterEach(async () => {+ await requestQueue.delete();+ });++ test('adding two requests without one being in the forefront should be added in sequential order', async () => {+ await requestQueue.addRequest({ url: 'http://example.com/1', uniqueKey: '1' });+ // Waiting a few ms is required since we use Date.now() to compute orderNo+ await sleep(2);+ await requestQueue.addRequest({ url: 'http://example.com/2', uniqueKey: '2' });++ const { items } = await requestQueue.listHead();++ expect(items).toHaveLength(2);+ expect(items[0].url).toBe('http://example.com/1');+ expect(items[1].url).toBe('http://example.com/2');+ });++ test('adding two requests with one being in the forefront should ensure the forefront request is first', async () => {+ await requestQueue.addRequest({ url: 'http://example.com/1', uniqueKey: '1' });+ // Waiting a few ms is required since we use Date.now() to compute orderNo+ await sleep(2);+ await requestQueue.addRequest({ url: 'http://example.com/2', uniqueKey: '2' }, { forefront: true });++ const { items } = await requestQueue.listHead();++ expect(items).toHaveLength(2);+ expect(items[0].url).toBe('http://example.com/2');+ expect(items[1].url).toBe('http://example.com/1');+ });++ test('adding two requests where both are in the forefront should ensure the latest one is added first', async () => {+ await requestQueue.addRequest({ url: 'http://example.com/1', uniqueKey: '1' }, { forefront: true });+ // Waiting a few ms is required since we use Date.now() to compute orderNo+ await sleep(2);+ await requestQueue.addRequest({ url: 'http://example.com/2', uniqueKey: '2' }, { forefront: true });++ const { items } = await requestQueue.listHead();++ expect(items).toHaveLength(2);+ expect(items[0].url).toBe('http://example.com/2');+ expect(items[1].url).toBe('http://example.com/1');+ });+});