Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Adding some extensions to COMMON_LOCAL_FILE_EXTENSIONS
Marek Trunkáte2cbf27ef4fe
diff --git a/src/key_value_store.js b/src/key_value_store.jsindex 965260722..a8328cc57 100644--- a/src/key_value_store.js+++ b/src/key_value_store.js@@ -13,7 +13,7 @@ import { addCharsetToContentType, apifyClient, ensureDirExists } from './utils';export const LOCAL_EMULATION_SUBDIR = LOCAL_EMULATION_SUBDIRS.keyValueStores;const MAX_OPENED_STORES = 1000;const DEFAULT_LOCAL_FILE_EXTENSION = 'bin';-const COMMON_LOCAL_FILE_EXTENSIONS = ['bin', 'txt', 'json', 'html', 'xml', 'jpeg', 'png', 'pdf', 'mp3', 'js', 'css'];+const COMMON_LOCAL_FILE_EXTENSIONS = ['bin', 'txt', 'json', 'html', 'xml', 'jpeg', 'png', 'pdf', 'mp3', 'js', 'css', 'xml', 'csv'];const readFilePromised = Promise.promisify(fs.readFile);const readdirPromised = Promise.promisify(fs.readdir);