Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Removing duplicate from COMMON_LOCAL_FILE_EXTENSIONS
Marek Trunkát9f65786f5369
diff --git a/src/key_value_store.js b/src/key_value_store.jsindex a8328cc57..1f26f84d5 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -13,7 +13,7 @@ import { addCharsetToContentType, apifyClient, ensureDirExists } from './utils';export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.keyValueStores;const MAX_OPENED_STORES = 1000;const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';-const COMMON_LOCAL_FILE_EXTENSIONS = ['bin', 'txt', 'json', 'html', 'xml', 'jpeg', 'png', 'pdf', 'mp3', 'js', 'css', 'xml', 'csv'];+const COMMON_LOCAL_FILE_EXTENSIONS = ['bin', 'txt', 'json', 'html', 'xml', 'jpeg', 'png', 'pdf', 'mp3', 'js', 'css', 'csv'];const readFilePromised = Promise.promisify(fs.readFile);const readdirPromised = Promise.promisify(fs.readdir);