Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add mime package to support all known content-type extensions

Ondra Urban47e69ffc1d55
diff --git a/package.json b/package.json
index 66c490795..f0b90a89e 100644
--- a/package.json
+++ b/package.json
@@ -51,6 +51,7 @@
"content-type": "^1.0.3",
"fs-extra": "^5.0.0",
"jquery": "^3.3.1",
+ "mime": "^2.3.1",
"pidusage": "=2.0.9",
"proxy-chain": "^0.1.26",
"ps-tree": "^1.1.0",
diff --git a/src/key_value_store.js b/src/key_value_store.js
index 6334765da..08cde511d 100644
--- a/src/key_value_store.js
+++ b/src/key_value_store.js
@@ -4,6 +4,7 @@ import path from 'path';
import Promise from 'bluebird';
import contentTypeParser from 'content-type';
import LruCache from 'apify-shared/lru_cache';
+import mime from 'mime';
import { KEY_VALUE_STORE_KEY_REGEX } from 'apify-shared/regexs';
import { checkParamOrThrow, parseBody } from 'apify-client/build/utils';
import { ENV_VARS, LOCAL_EMULATION_SUBDIRS } from './constants';
@@ -11,17 +12,10 @@ import { addCharsetToContentType, apifyClient, ensureDirExists } from './utils';
export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.keyValueStores;
const MAX_OPENED_STORES = 1000;
-const LOCAL_FILE_TYPES = [
- { contentType: 'application/octet-stream', extension: 'buffer' },
- { contentType: 'application/json', extension: 'json' },
- { contentType: 'text/plain', extension: 'txt' },
- { contentType: 'image/jpeg', extension: 'jpg' },
- { contentType: 'image/png', extension: 'png' },
- { contentType: 'application/xml', extension: 'xml' },
-];
-const DEFAULT_LOCAL_FILE_TYPE = LOCAL_FILE_TYPES[0];
+const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';
const readFilePromised = Promise.promisify(fs.readFile);
+const readdirPromised = Promise.promisify(fs.readdir);
const writeFilePromised = Promise.promisify(fs.writeFile);
const unlinkPromised = Promise.promisify(fs.unlink);
const emptyDirPromised = Promise.promisify(fsExtra.emptyDir);
@@ -207,26 +201,19 @@ export class KeyValueStoreLocal {
validateGetValueParams(key);
return this.initializationPromise
- .then(() => {
- const filePath = path.resolve(this.localEmulationPath, key);
- const promises = LOCAL_FILE_TYPES.map(({ extension }) => {
- return readFilePromised(`${filePath}.${extension}`).catch(() => null);
- });
-
- return Promise.all(promises);
- })
+ .then(() => readdirPromised(this.localEmulationPath))
.then((files) => {
- let body = null;
-
- LOCAL_FILE_TYPES.some(({ contentType }, index) => {
- if (files[index] !== null) {
- body = parseBody(files[index], contentType);
-
- return true;
- }
- });
-
- return body;
+ const regex = new RegExp(`^${key}\\.[a-z0-9]+$`);
+ const fileName = files.find(file => regex.test(file));
+ if (fileName) {
+ const filePath = path.resolve(this.localEmulationPath, fileName);
+ return readFilePromised(filePath)
+ .then((fileBuffer) => {
+ const contentType = mime.getType(fileName);
+ return parseBody(fileBuffer, contentType);
+ });
+ }
+ return null;
})
.catch((err) => {
throw new Error(`Error reading file '${key}' in directory '${this.localEmulationPath}' referred by ${ENV_VARS.APIFY_LOCAL_EMULATION_DIR} environment variable: ${err.message}`); // eslint-disable-line max-len
@@ -235,21 +222,17 @@ export class KeyValueStoreLocal {
setValue(key, value, options = {}) {
validateSetValueParams(key, value, options);
+ const getPath = fileName => path.resolve(this.localEmulationPath, fileName);
// Make copy of options, don't update what user passed.
const optionsCopy = Object.assign({}, options);
- const deletePromisesArr = LOCAL_FILE_TYPES.map(({ extension }) => {
- const filePath = path.resolve(this.localEmulationPath, `${key}.${extension}`);
-
- return unlinkPromised(filePath)
- .catch((err) => {
- if (err.code !== 'ENOENT') throw err;
- });
- });
-
- const deletePromise = Promise
- .all(deletePromisesArr);
+ const deletePromise = readdirPromised(this.localEmulationPath)
+ .then((files) => {
+ const regex = new RegExp(`^${key}\\.[a-z0-9]+$`);
+ const fileName = files.find(file => regex.test(file));
+ return fileName ? unlinkPromised(getPath(fileName)) : Promise.resolve();
+ });
// In this case delete the record.
if (value === null) return deletePromise;
@@ -257,8 +240,8 @@ export class KeyValueStoreLocal {
value = maybeStringify(value, optionsCopy);
const contentType = contentTypeParser.parse(optionsCopy.contentType).type;
- const { extension } = LOCAL_FILE_TYPES.filter(type => type.contentType === contentType).pop() || DEFAULT_LOCAL_FILE_TYPE;
- const filePath = path.resolve(this.localEmulationPath, `${key}.${extension}`);
+ const extension = mime.getExtension(contentType) || DEFAULT_LOCAL_FILE_EXTENSION;
+ const filePath = getPath(`${key}.${extension}`);
return deletePromise
.then(() => writeFilePromised(filePath, value))