Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Updated BLOCK_TAGS_REGEX
Jan Curn7c6a6a03db92
diff --git a/src/utils.js b/src/utils.jsindex bc678f1d5..b522478b8 100644--- a/src/utils.js+++ b/src/utils.js@@ -485,7 +485,7 @@ export const ensureTokenOrLocalStorageEnvExists = (storageName) => {// NOTE: We skipping 'noscript' since it's content is evaluated as text, instead of HTML elements. That damages the results.const SKIP_TAGS_REGEX = /^(script|style|canvas|svg|noscript)$/i;-const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|div|fieldset|form|table|tr)$/i;+const BLOCK_TAGS_REGEX = /^(p|h1|h2|h3|h4|h5|h6|ol|ul|li|pre|address|blockquote|dl|div|fieldset|form|table|tr|select|option)$/i;/**