Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Fix/apify shared (#63)
Jan Čurn435f6bd1d0bd
diff --git a/package.json b/package.jsonindex e43d4e1e5..b6d5438fa 100644--- a/package.json+++ b/package.json@@ -44,7 +44,7 @@},"dependencies": {"apify-client": "^0.2.7",- "apify-shared": "0.0.25",+ "apify-shared": "^0.0.25","bluebird": "^3.5.0","content-type": "^1.0.3","fs-extra": "^5.0.0",diff --git a/src/request_list.js b/src/request_list.jsindex 384f5e3f4..df6499712 100644--- a/src/request_list.js+++ b/src/request_list.js@@ -11,6 +11,8 @@ import { getFirstKey } from './utils';import { getValue, setValue } from './key_value_store';// TODO: better tests+// TODO: this will not accept diacritict chars, IMHO we should have here a regexp+// that detects any hostname/whatever stringconst URL_REGEX = '(http|https)://[\\w-]+(\\.[\\w-]+)+([\\w-.,@?^=%&:/~+#-]*[\\w@?^=%&;/~+#-])?';/**