Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix(MemoryStorage): correctly respect the desc option (#1666)

Vlad Frangub5f37f66a50b
diff --git a/packages/memory-storage/src/resource-clients/dataset.ts b/packages/memory-storage/src/resource-clients/dataset.ts
index 189562b03..d0dd3494d 100644
--- a/packages/memory-storage/src/resource-clients/dataset.ts
+++ b/packages/memory-storage/src/resource-clients/dataset.ts
@@ -132,7 +132,10 @@ export class DatasetClient<Data extends Dictionary = Dictionary> extends BaseCli
this.throwOnNonExisting(StorageTypes.Dataset);
}
- const [start, end] = existingStoreById.getStartAndEndIndexes(offset, limit);
+ const [start, end] = existingStoreById.getStartAndEndIndexes(
+ desc ? Math.max(existingStoreById.itemCount - offset - limit, 0) : offset,
+ limit,
+ );
const items: Data[] = [];
diff --git a/packages/memory-storage/test/reverse-datataset-list.test.ts b/packages/memory-storage/test/reverse-datataset-list.test.ts
new file mode 100644
index 000000000..306ba8c72
--- /dev/null
+++ b/packages/memory-storage/test/reverse-datataset-list.test.ts
@@ -0,0 +1,55 @@
+import { rm } from 'node:fs/promises';
+import { resolve } from 'node:path';
+import { MemoryStorage } from '@crawlee/memory-storage';
+import type { DatasetClient } from '@crawlee/types';
+
+const elements = Array.from({ length: 10 }, (_, i) => ({ number: i }));
+
+describe('Dataset#listItems respects the desc option', () => {
+ const localDataDirectory = resolve(__dirname, './tmp/desc');
+ const storage = new MemoryStorage({
+ localDataDirectory,
+ persistStorage: false,
+ });
+
+ let dataset: DatasetClient;
+
+ afterAll(async () => {
+ await rm(localDataDirectory, { force: true, recursive: true });
+ });
+
+ beforeAll(async () => {
+ const { id: falseDatasetId } = await storage.datasets().getOrCreate('false');
+ dataset = storage.dataset(falseDatasetId);
+
+ await dataset.pushItems(elements);
+ });
+
+ test('with desc: false', async () => {
+ const result = await dataset.listItems({ desc: false, limit: 5 });
+
+ expect(result.items).toHaveLength(5);
+ expect(result.items).toStrictEqual(elements.slice(0, 5));
+ });
+
+ test('with desc: true', async () => {
+ const result = await dataset.listItems({ desc: true, limit: 5 });
+
+ expect(result.items).toHaveLength(5);
+ expect(result.items).toStrictEqual(elements.slice().reverse().slice(0, 5));
+ });
+
+ test('with desc: false and offset: 2', async () => {
+ const result = await dataset.listItems({ desc: false, limit: 5, offset: 2 });
+
+ expect(result.items).toHaveLength(5);
+ expect(result.items).toStrictEqual(elements.slice(2, 7));
+ });
+
+ test('with desc: true and offset: 2', async () => {
+ const result = await dataset.listItems({ desc: true, limit: 5, offset: 2 });
+
+ expect(result.items).toHaveLength(5);
+ expect(result.items).toStrictEqual(elements.slice().reverse().slice(2, 7));
+ });
+});