Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
fix: improve fix for double extension in KVS with HTML files (#2505)
Martin Adámek157927d67f42
diff --git a/packages/memory-storage/src/fs/key-value-store/fs.ts b/packages/memory-storage/src/fs/key-value-store/fs.tsindex 64a67571c..c08e1a0c4 100644--- a/packages/memory-storage/src/fs/key-value-store/fs.ts+++ b/packages/memory-storage/src/fs/key-value-store/fs.ts@@ -62,7 +62,12 @@ export class KeyValueFileSystemEntry implements StorageImplementation<InternalKeasync update(data: InternalKeyRecord) {await this.fsQueue.wait();- const fileName = mime.contentType(data.key) === data.contentType ? data.key : `${data.key}.${data.extension}`;+ const contentType = mime.contentType(data.key);+ const fileName =+ // the content type might include charset, e.g. `text/html; charset=utf-8`, so we check via `startsWith` instead of `===`+ contentType && data.contentType && contentType.startsWith(data.contentType)+ ? data.key+ : `${data.key}.${data.extension}`;this.filePath ??= resolve(this.storeDirectory, fileName);this.fileMetadataPath ??= resolve(this.storeDirectory, `${data.key}.__metadata__.json`);diff --git a/packages/memory-storage/test/key-value-store/with-extension.test.ts b/packages/memory-storage/test/key-value-store/with-extension.test.tsindex 3fb5990a0..50288ee47 100644--- a/packages/memory-storage/test/key-value-store/with-extension.test.ts+++ b/packages/memory-storage/test/key-value-store/with-extension.test.ts@@ -8,7 +8,7 @@ describe('KeyValueStore should append extension only when needed', () => {afterAll(() => emptyDirSync('tmp'));- test('should append extension when needed', async () => {+ test('should append extension when needed (jpg)', async () => {const testDir = resolve('tmp', 'test_no_extension');const storage = createKeyValueStorageImplementation({persistStorage: true,@@ -26,6 +26,24 @@ describe('KeyValueStore should append extension only when needed', () => {expect(existsSync(resolve(testDir, 'jibberish'))).toBeFalsy();});+ test('should append extension when needed (html)', async () => {+ const testDir = resolve('tmp', 'test_no_extension');+ const storage = createKeyValueStorageImplementation({+ persistStorage: true,+ storeDirectory: testDir,+ writeMetadata: true,+ });+ await storage.update({+ key: 'jibberish2',+ value: '<html lang="en"><body>Hi there!</body></html>',+ contentType: 'text/html',+ extension: 'html',+ });++ expect(existsSync(resolve(testDir, 'jibberish2.html'))).toBeTruthy();+ expect(existsSync(resolve(testDir, 'jibberish2'))).toBeFalsy();+ });+test('should not append extension when already available', async () => {const testDir = resolve('tmp', 'test_extension');const storage = createKeyValueStorageImplementation({@@ -43,4 +61,22 @@ describe('KeyValueStore should append extension only when needed', () => {expect(existsSync(resolve(testDir, 'jibberish.jpg'))).toBeTruthy();expect(existsSync(resolve(testDir, 'jibberish.jpg.jpeg'))).toBeFalsy();});++ test('should not append extension when already available', async () => {+ const testDir = resolve('tmp', 'test_extension');+ const storage = createKeyValueStorageImplementation({+ persistStorage: true,+ storeDirectory: testDir,+ writeMetadata: true,+ });+ await storage.update({+ key: 'jibberish2.html',+ value: '<html lang="en"><body>Hi there!</body></html>',+ contentType: 'text/html',+ extension: 'html',+ });++ expect(existsSync(resolve(testDir, 'jibberish2.html'))).toBeTruthy();+ expect(existsSync(resolve(testDir, 'jibberish2.html.html'))).toBeFalsy();+ });});