Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Fix/apify shared (#63)

Jan Čurn435f6bd1d0bd
diff --git a/package.json b/package.json
index e43d4e1e5..b6d5438fa 100644
--- a/package.json
+++ b/package.json
@@ -44,7 +44,7 @@
},
"dependencies": {
"apify-client": "^0.2.7",
- "apify-shared": "0.0.25",
+ "apify-shared": "^0.0.25",
"bluebird": "^3.5.0",
"content-type": "^1.0.3",
"fs-extra": "^5.0.0",
diff --git a/src/request_list.js b/src/request_list.js
index 384f5e3f4..df6499712 100644
--- a/src/request_list.js
+++ b/src/request_list.js
@@ -11,6 +11,8 @@ import { getFirstKey } from './utils';
import { getValue, setValue } from './key_value_store';
// TODO: better tests
+// TODO: this will not accept diacritict chars, IMHO we should have here a regexp
+// that detects any hostname/whatever string
const URL_REGEX = '(http|https)://[\\w-]+(\\.[\\w-]+)+([\\w-.,@?^=%&:/~+#-]*[\\w@?^=%&;/~+#-])?';
/**