Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix(MemoryStorage): correctly respect the desc option (#1666)
Vlad Frangub5f37f66a50b
diff --git a/packages/memory-storage/src/resource-clients/dataset.ts b/packages/memory-storage/src/resource-clients/dataset.tsindex 189562b03..d0dd3494d 100644--- a/packages/memory-storage/src/resource-clients/dataset.ts+++ b/packages/memory-storage/src/resource-clients/dataset.ts@@ -132,7 +132,10 @@ export class DatasetClient<Data extends Dictionary = Dictionary> extends BaseClithis.throwOnNonExisting(StorageTypes.Dataset);}- const [start, end] = existingStoreById.getStartAndEndIndexes(offset, limit);+ const [start, end] = existingStoreById.getStartAndEndIndexes(+ desc ? Math.max(existingStoreById.itemCount - offset - limit, 0) : offset,+ limit,+ );const items: Data[] = [];diff --git a/packages/memory-storage/test/reverse-datataset-list.test.ts b/packages/memory-storage/test/reverse-datataset-list.test.tsnew file mode 100644index 000000000..306ba8c72--- /dev/null+++ b/packages/memory-storage/test/reverse-datataset-list.test.ts@@ -0,0 +1,55 @@+import { rm } from 'node:fs/promises';+import { resolve } from 'node:path';+import { MemoryStorage } from '@crawlee/memory-storage';+import type { DatasetClient } from '@crawlee/types';++const elements = Array.from({ length: 10 }, (_, i) => ({ number: i }));++describe('Dataset#listItems respects the desc option', () => {+ const localDataDirectory = resolve(__dirname, './tmp/desc');+ const storage = new MemoryStorage({+ localDataDirectory,+ persistStorage: false,+ });++ let dataset: DatasetClient;++ afterAll(async () => {+ await rm(localDataDirectory, { force: true, recursive: true });+ });++ beforeAll(async () => {+ const { id: falseDatasetId } = await storage.datasets().getOrCreate('false');+ dataset = storage.dataset(falseDatasetId);++ await dataset.pushItems(elements);+ });++ test('with desc: false', async () => {+ const result = await dataset.listItems({ desc: false, limit: 5 });++ expect(result.items).toHaveLength(5);+ expect(result.items).toStrictEqual(elements.slice(0, 5));+ });++ test('with desc: true', async () => {+ const result = await dataset.listItems({ desc: true, limit: 5 });++ expect(result.items).toHaveLength(5);+ expect(result.items).toStrictEqual(elements.slice().reverse().slice(0, 5));+ });++ test('with desc: false and offset: 2', async () => {+ const result = await dataset.listItems({ desc: false, limit: 5, offset: 2 });++ expect(result.items).toHaveLength(5);+ expect(result.items).toStrictEqual(elements.slice(2, 7));+ });++ test('with desc: true and offset: 2', async () => {+ const result = await dataset.listItems({ desc: true, limit: 5, offset: 2 });++ expect(result.items).toHaveLength(5);+ expect(result.items).toStrictEqual(elements.slice().reverse().slice(2, 7));+ });+});