Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: Fixed double extension for screenshots (#2419)
Robert-Andrei Damiane8b39c417647
diff --git a/packages/memory-storage/src/fs/key-value-store/fs.ts b/packages/memory-storage/src/fs/key-value-store/fs.tsindex 3a9443a34..fe62f6aa6 100644--- a/packages/memory-storage/src/fs/key-value-store/fs.ts+++ b/packages/memory-storage/src/fs/key-value-store/fs.ts@@ -4,6 +4,7 @@ import { basename } from 'node:path/win32';import { AsyncQueue } from '@sapphire/async-queue';import { ensureDir } from 'fs-extra';+import mime from 'mime-types';import { lockAndWrite } from '../../background-handler/fs-utils';import type { InternalKeyRecord } from '../../resource-clients/key-value-store';@@ -57,7 +58,9 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKeasync update(data: InternalKeyRecord) {await this.fsQueue.wait();- this.filePath ??= resolve(this.storeDirectory, `${data.key}.${data.extension}`);+ const fileName = mime.contentType(data.key) === data.contentType ? data.key : `${data.key}.${data.extension}`;++ this.filePath ??= resolve(this.storeDirectory, fileName);this.fileMetadataPath ??= resolve(this.storeDirectory, `${data.key}.__metadata__.json`);const { value, ...rest } = data;diff --git a/packages/memory-storage/test/key-value-store-stream.test.ts b/packages/memory-storage/test/key-value-store/stream.test.tssimilarity index 100%rename from packages/memory-storage/test/key-value-store-stream.test.tsrename to packages/memory-storage/test/key-value-store/stream.test.tsdiff --git a/packages/memory-storage/test/key-value-store/with-extension.test.ts b/packages/memory-storage/test/key-value-store/with-extension.test.tsnew file mode 100644index 000000000..e551573f9--- /dev/null+++ b/packages/memory-storage/test/key-value-store/with-extension.test.ts@@ -0,0 +1,36 @@+import { resolve } from 'node:path';++import { existsSync, emptyDirSync } from 'fs-extra';+import { createKeyValueStorageImplementation } from 'packages/memory-storage/src/fs/key-value-store';++describe('KeyValueStore should append extension only when needed', () => {+ const mockImageBuffer = Buffer.from('This is a test image', 'utf8');++ afterAll(() => emptyDirSync('tmp'));++ test('should append extension when needed', async () => {+ const testDir = resolve('tmp', 'test_no_extension');+ const storage = createKeyValueStorageImplementation({+ persistStorage: true,+ storeDirectory: testDir,+ writeMetadata: true,+ });+ await storage.update({ key: 'jibberish', value: mockImageBuffer, contentType: 'image/jpeg', extension: 'jpeg' });++ expect(existsSync(resolve(testDir, 'jibberish.jpeg'))).toBeTruthy();+ expect(existsSync(resolve(testDir, 'jibberish'))).toBeFalsy();+ });++ test('should not append extension when already available', async () => {+ const testDir = resolve('tmp', 'test_extension');+ const storage = createKeyValueStorageImplementation({+ persistStorage: true,+ storeDirectory: testDir,+ writeMetadata: true,+ });+ await storage.update({ key: 'jibberish.jpg', value: mockImageBuffer, contentType: 'image/jpeg', extension: 'jpeg' });++ expect(existsSync(resolve(testDir, 'jibberish.jpg'))).toBeTruthy();+ expect(existsSync(resolve(testDir, 'jibberish.jpg.jpeg'))).toBeFalsy();+ });+});