Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Merge pull request #401 from vdrmota/feature/infiniteScroll
Ondra Urbaneb0183d82c00
diff --cc docs/api/utils.mdindex 96b3e1b37,ddf13066f..61c93a2a8--- a/docs/api/utils.md+++ b/docs/api/utils.md@@@ -23,9 -23,10 +23,10 @@@ await Apify.utils.sleep(1500)* [`.sleep`](#utils.sleep) ⇒ `Promise`* [`.URL_NO_COMMAS_REGEX`](#utils.URL_NO_COMMAS_REGEX)* [`.URL_WITH_COMMAS_REGEX`](#utils.URL_WITH_COMMAS_REGEX)+ * [`.exports.infiniteScroll`](#utils.exports.infiniteScroll) ⇒ `Promise`* [`.isDocker()`](#utils.isDocker) ⇒ `Promise`* [`.downloadListOfUrls(options)`](#utils.downloadListOfUrls) ⇒ `Promise<Array<String>>`- * [`.extractUrls(string, [urlRegExp])`](#utils.extractUrls) ⇒ `Array<String>`+ * [`.extractUrls(options)`](#utils.extractUrls) ⇒ `Array<String>`* [`.getRandomUserAgent()`](#utils.getRandomUserAgent) ⇒ `String`* [`.htmlToText(html)`](#utils.htmlToText) ⇒ `String`diff --cc package.jsonindex 091ec4858,d4f16ce1f..cc607ca12--- a/package.json+++ b/package.json@@@ -42,15 -42,12 +42,16 @@@"build-docs": "npm run clean && npm run build && node ./website/tools/build_docs.js","build-readme": "node ./tools/build_readme.js","test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --require @babel/register --recursive --exit",+ "test:integration": "npm run build && nyc --reporter=html --reporter=text mocha 'test/integration_tests' --timeout 60000 --require @babel/register --recursive --exit","prepare": "npm run build",- "prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint",+ "prepublishOnly": "(test $TRAVIS || (echo \"Publishing is reserved to the Travis CI!\"; exit 1))","clean": "rm -rf build","lint": "npm run build && ./node_modules/.bin/eslint ./src ./test",- "lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix"+ "lint:fix": "./node_modules/.bin/eslint ./src ./test --ext .js,.jsx --fix",+ "before-deploy": "node .travis/before-deploy.js",+ "after-deploy": "node after-deploy.js",+ "prerelease-check": "npm run lint && node check-docs.js",+ "release-latest": "node release-latest.js"},"dependencies": {"@apify/ps-tree": "^1.1.3",