Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fix: Fixed double extension for screenshots (#2419)

Robert-Andrei Damiane8b39c417647
diff --git a/packages/memory-storage/src/fs/key-value-store/fs.ts b/packages/memory-storage/src/fs/key-value-store/fs.ts
index 3a9443a34..fe62f6aa6 100644
--- a/packages/memory-storage/src/fs/key-value-store/fs.ts
+++ b/packages/memory-storage/src/fs/key-value-store/fs.ts
@@ -4,6 +4,7 @@ import { basename } from 'node:path/win32';
import { AsyncQueue } from '@sapphire/async-queue';
import { ensureDir } from 'fs-extra';
+import mime from 'mime-types';
import { lockAndWrite } from '../../background-handler/fs-utils';
import type { InternalKeyRecord } from '../../resource-clients/key-value-store';
@@ -57,7 +58,9 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKe
async update(data: InternalKeyRecord) {
await this.fsQueue.wait();
- this.filePath ??= resolve(this.storeDirectory, `${data.key}.${data.extension}`);
+ const fileName = mime.contentType(data.key) === data.contentType ? data.key : `${data.key}.${data.extension}`;
+
+ this.filePath ??= resolve(this.storeDirectory, fileName);
this.fileMetadataPath ??= resolve(this.storeDirectory, `${data.key}.__metadata__.json`);
const { value, ...rest } = data;
diff --git a/packages/memory-storage/test/key-value-store-stream.test.ts b/packages/memory-storage/test/key-value-store/stream.test.ts
similarity index 100%
rename from packages/memory-storage/test/key-value-store-stream.test.ts
rename to packages/memory-storage/test/key-value-store/stream.test.ts
diff --git a/packages/memory-storage/test/key-value-store/with-extension.test.ts b/packages/memory-storage/test/key-value-store/with-extension.test.ts
new file mode 100644
index 000000000..e551573f9
--- /dev/null
+++ b/packages/memory-storage/test/key-value-store/with-extension.test.ts
@@ -0,0 +1,36 @@
+import { resolve } from 'node:path';
+
+import { existsSync, emptyDirSync } from 'fs-extra';
+import { createKeyValueStorageImplementation } from 'packages/memory-storage/src/fs/key-value-store';
+
+describe('KeyValueStore should append extension only when needed', () => {
+ const mockImageBuffer = Buffer.from('This is a test image', 'utf8');
+
+ afterAll(() => emptyDirSync('tmp'));
+
+ test('should append extension when needed', async () => {
+ const testDir = resolve('tmp', 'test_no_extension');
+ const storage = createKeyValueStorageImplementation({
+ persistStorage: true,
+ storeDirectory: testDir,
+ writeMetadata: true,
+ });
+ await storage.update({ key: 'jibberish', value: mockImageBuffer, contentType: 'image/jpeg', extension: 'jpeg' });
+
+ expect(existsSync(resolve(testDir, 'jibberish.jpeg'))).toBeTruthy();
+ expect(existsSync(resolve(testDir, 'jibberish'))).toBeFalsy();
+ });
+
+ test('should not append extension when already available', async () => {
+ const testDir = resolve('tmp', 'test_extension');
+ const storage = createKeyValueStorageImplementation({
+ persistStorage: true,
+ storeDirectory: testDir,
+ writeMetadata: true,
+ });
+ await storage.update({ key: 'jibberish.jpg', value: mockImageBuffer, contentType: 'image/jpeg', extension: 'jpeg' });
+
+ expect(existsSync(resolve(testDir, 'jibberish.jpg'))).toBeTruthy();
+ expect(existsSync(resolve(testDir, 'jibberish.jpg.jpeg'))).toBeFalsy();
+ });
+});