Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
options object
vdrmotae5d65fad18c2
diff --git a/report.20190619.113804.50671.001.json b/report.20190619.113804.50671.001.jsondeleted file mode 100644index 848ecaca7..000000000--- a/report.20190619.113804.50671.001.json+++ /dev/null@@ -1,671 +0,0 @@--{- "header": {- "event": "Allocation failed - JavaScript heap out of memory",- "trigger": "FatalError",- "filename": "report.20190619.113804.50671.001.json",- "dumpEventTime": "2019-06-19T11:38:04Z",- "dumpEventTimeStamp": "1560937084495",- "processId": 50671,- "commandLine": [- "/usr/local/Cellar/node/11.12.0/bin/node",- "/Users/vojtad/.node-spawn-wrap-50669-919223399a2c/node",- "/Users/vojtad/Documents/Apify/apify-js/node_modules/mocha/bin/_mocha",- "--timeout",- "60000",- "--require",- "@babel/register",- "--recursive",- "--exit"- ],- "nodejsVersion": "v11.12.0",- "wordSize": 64,- "arch": "x64",- "platform": "darwin",- "componentVersions": {- "node": "11.12.0",- "v8": "7.0.276.38-node.18",- "uv": "1.26.0",- "zlib": "1.2.11",- "brotli": "1.0.7",- "ares": "1.15.0",- "modules": "67",- "nghttp2": "1.34.0",- "napi": "4",- "llhttp": "1.1.1",- "http_parser": "2.8.0",- "openssl": "1.1.1b",- "cldr": "34.0",- "icu": "63.1",- "tz": "2018e",- "unicode": "11.0"- },- "release": {- "name": "node",- "headersUrl": "https://nodejs.org/download/release/v11.12.0/node-v11.12.0-headers.tar.gz",- "sourceUrl": "https://nodejs.org/download/release/v11.12.0/node-v11.12.0.tar.gz"- },- "osName": "Darwin",- "osRelease": "17.7.0",- "osVersion": "Darwin Kernel Version 17.7.0: Thu Jun 21 22:53:14 PDT 2018; root:xnu-4570.71.2~1/RELEASE_X86_64",- "osMachine": "x86_64",- "host": "Vojtas-MacBook-Air.local"- },- "javascriptStack": {- "message": "No stack.",- "stack": [- "Unavailable."- ]- },- "nativeStack": [- {- "pc": "0x0000000100127384",- "symbol": "report::TriggerNodeReport(v8::Isolate*, node::Environment*, char const*, char const*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, v8::Local<v8::String>) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x0000000100064c2a",- "symbol": "node::OnFatalError(char const*, char const*) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x000000010017e40f",- "symbol": "v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x000000010017e3b0",- "symbol": "v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char const*, bool) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x0000000100440714",- "symbol": "v8::internal::Heap::UpdateSurvivalStatistics(int) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x0000000100442131",- "symbol": "v8::internal::Heap::CheckIneffectiveMarkCompact(unsigned long, double) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x000000010043faa9",- "symbol": "v8::internal::Heap::PerformGarbageCollection(v8::internal::GarbageCollector, v8::GCCallbackFlags) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x000000010043e89f",- "symbol": "v8::internal::Heap::CollectGarbage(v8::internal::AllocationSpace, v8::internal::GarbageCollectionReason, v8::GCCallbackFlags) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x000000010044676d",- "symbol": "v8::internal::Heap::AllocateRawWithLightRetry(int, v8::internal::AllocationSpace, v8::internal::AllocationAlignment) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x00000001004467bc",- "symbol": "v8::internal::Heap::AllocateRawWithRetryOrFail(int, v8::internal::AllocationSpace, v8::internal::AllocationAlignment) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x0000000100426b83",- "symbol": "v8::internal::Factory::NewFillerObject(int, bool, v8::internal::AllocationSpace) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x0000000100603f9e",- "symbol": "v8::internal::Runtime_AllocateInNewSpace(int, v8::internal::Object**, v8::internal::Isolate*) [/usr/local/Cellar/node/11.12.0/bin/node]"- },- {- "pc": "0x00003c15fcc4fc7d",- "symbol": ""- }- ],- "javascriptHeap": {- "totalMemory": 1473159168,- "totalCommittedMemory": 1470533744,- "usedMemory": 1441614376,- "availableMemory": 55362560,- "memoryLimit": 1526909922,- "heapSpaces": {- "read_only_space": {- "memorySize": 524288,- "committedMemory": 42224,- "capacity": 515584,- "used": 33520,- "available": 482064- },- "new_space": {- "memorySize": 2097152,- "committedMemory": 1119392,- "capacity": 1031168,- "used": 53568,- "available": 977600- },- "old_space": {- "memorySize": 1460342784,- "committedMemory": 1459907712,- "capacity": 1434753560,- "used": 1434409448,- "available": 344112- },- "code_space": {- "memorySize": 2621440,- "committedMemory": 2108096,- "capacity": 1921888,- "used": 1921888,- "available": 0- },- "map_space": {- "memorySize": 2633728,- "committedMemory": 2416544,- "capacity": 1623680,- "used": 1623680,- "available": 0- },- "large_object_space": {- "memorySize": 4939776,- "committedMemory": 4939776,- "capacity": 57131056,- "used": 3572272,- "available": 53558784- },- "new_large_object_space": {- "memorySize": 0,- "committedMemory": 0,- "capacity": 0,- "used": 0,- "available": 0- }- }- },- "resourceUsage": {- "userCpuSeconds": 115.251,- "kernelCpuSeconds": 5.14185,- "cpuConsumptionPercent": 120.393,- "maxRss": 1554782355456,- "pageFaults": {- "IORequired": 71,- "IONotRequired": 804958- },- "fsActivity": {- "reads": 0,- "writes": 0- }- },- "libuv": [- ],- "environmentVariables": {- "npm_config_save_dev": "",- "npm_config_legacy_bundling": "",- "npm_config_dry_run": "",- "npm_package_devDependencies_chai_subset": "^1.6.0",- "npm_package_dependencies_request": "^2.88.0",- "npm_package_dependencies__apify_ps_tree": "^1.1.3",- "npm_config_viewer": "man",- "npm_config_only": "",- "npm_config_commit_hooks": "true",- "npm_config_browser": "",- "npm_package_gitHead": "769598b4f8dd62ca2be0e0e580636b2a05e0eeb9",- "npm_config_also": "",- "npm_package_dependencies_apify_client": "^0.5.19",- "npm_config_sign_git_commit": "",- "npm_config_rollback": "true",- "TERM_PROGRAM": "vscode",- "NODE": "/usr/local/Cellar/node/11.12.0/bin/node",- "npm_config_usage": "",- "npm_config_audit": "true",- "npm_package_devDependencies_socket_io_client": "^2.2.0",- "npm_package_devDependencies_fpcollect": "^1.0.4",- "npm_package_dependencies_cheerio": "^1.0.0-rc.2",- "INIT_CWD": "/Users/vojtad/Documents/Apify/apify-js",- "npm_package_devDependencies_sinon": "^7.1.1",- "npm_package_devDependencies__types_puppeteer": "^1.12.4",- "npm_package_dependencies_proxy_chain": "^0.2.5",- "npm_package_dependencies_iltorb": "^2.4.2",- "npm_package_homepage": "https://sdk.apify.com/",- "ANDROID_HOME": "/Users/vojtad/Library/Android/sdk",- "npm_config_globalignorefile": "/usr/local/etc/npmignore",- "npm_package_devDependencies_mocha": "^5.2.0",- "TERM": "xterm-256color",- "SHELL": "/bin/bash",- "npm_config_shell": "/bin/bash",- "npm_config_maxsockets": "50",- "npm_config_init_author_url": "",- "npm_package_devDependencies_portastic": "^1.0.1",- "npm_package_devDependencies_basic_auth_parser": "^0.0.2",- "npm_package_dependencies_htmlparser2": "^3.10.1",- "npm_config_shrinkwrap": "true",- "npm_config_parseable": "",- "npm_config_metrics_registry": "https://registry.npmjs.org/",- "npm_package_dependencies_xregexp": "^4.2.4",- "TMPDIR": "/var/folders/sw/01254g6n3nz7hbd9vx6slg840000gn/T/",- "npm_config_timing": "",- "npm_config_init_license": "ISC",- "npm_package_dependencies_socket_io": "^2.2.0",- "npm_package_scripts_lint": "npm run build && ./node_modules/.bin/eslint ./src ./test",- "Apple_PubSub_Socket_Render": "/private/tmp/com.apple.launchd.EPJ9mKsnYj/Render",- "npm_config_if_present": "",- "npm_package_dependencies_rimraf": "^2.6.2",- "TERM_PROGRAM_VERSION": "1.34.0",- "npm_config_sign_git_tag": "",- "npm_config_init_author_email": "",- "npm_config_cache_max": "Infinity",- "npm_package_optionalDependencies_selenium_webdriver": "^3.6.0",- "npm_package_scripts_prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint",- "npm_config_preid": "",- "npm_config_long": "",- "npm_config_local_address": "",- "npm_config_git_tag_version": "true",- "npm_config_cert": "",- "TERM_SESSION_ID": "D0D76223-185D-43EB-B4FA-41E0940819D9",- "npm_config_registry": "https://registry.npmjs.org/",- "npm_config_noproxy": "",- "npm_config_fetch_retries": "2",- "npm_package_repository_url": "git+https://github.com/apifytech/apify-js.git",- "npm_config_versions": "",- "npm_config_message": "%s",- "npm_config_key": "",- "npm_package_readmeFilename": "README.md",- "npm_package_devDependencies_why_is_node_running": "^2.0.3",- "npm_package_devDependencies_eslint_plugin_jsx_a11y": "^6.1.2",- "npm_package_description": "The scalable web crawling and scraping library for JavaScript. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",- "USER": "vojtad",- "npm_package_license": "Apache-2.0",- "npm_config_globalconfig": "/usr/local/etc/npmrc",- "npm_package_devDependencies_proxy": "^0.2.4",- "npm_package_dependencies_request_promise_native": "^1.0.5",- "npm_package_dependencies_apify_shared": "^0.1.51",- "npm_config_prefer_online": "",- "npm_config_logs_max": "10",- "npm_config_always_auth": "",- "npm_package_devDependencies__babel_core": "^7.0.0",- "npm_package_contributors_2_name": "Ondra Urban",- "SSH_AUTH_SOCK": "/private/tmp/com.apple.launchd.Gh7whWq4Uh/Listeners",- "npm_package_devDependencies_eslint": "^5.9.0",- "__CF_USER_TEXT_ENCODING": "0x1F5:0x0:0x0",- "npm_execpath": "/usr/local/lib/node_modules/npm/bin/npm-cli.js",- "npm_config_global_style": "",- "npm_config_cache_lock_retries": "10",- "npm_config_update_notifier": "true",- "npm_config_cafile": "",- "npm_package_optionalDependencies_puppeteer": "1.17.0",- "npm_package_author_name": "Apify",- "npm_config_heading": "npm",- "npm_config_audit_level": "low",- "npm_config_searchlimit": "20",- "npm_config_read_only": "",- "npm_config_offline": "",- "npm_config_fetch_retry_mintimeout": "10000",- "npm_config_json": "",- "npm_config_access": "",- "npm_config_argv": "{\"remain\":[],\"cooked\":[\"run\",\"test\"],\"original\":[\"run\",\"test\"]}",- "npm_package_devDependencies__types_socket_io": "^2.1.2",- "npm_package_dependencies_underscore": "^1.9.1",- "npm_package_scripts_lint_fix": "eslint ./src ./test --ext .js,.jsx --fix",- "PATH": "/usr/local/lib/node_modules/npm/node_modules/npm-lifecycle/node-gyp-bin:/Users/vojtad/Documents/Apify/apify-js/node_modules/.bin:/Library/Frameworks/Python.framework/Versions/3.6/bin:/Users/vojtad/.rbenv/shims:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin:/Library/Frameworks/Python.framework/Versions/3.6/bin:/Users/vojtad/.rbenv/shims:/Users/vojtad/Library/Android/sdk/emulator:/Users/vojtad/Library/Android/sdk/tools:/Users/vojtad/Library/Android/sdk/tools/bin:/Users/vojtad/Library/Android/sdk/platform-tools:/Users/vojtad/Library/Android/sdk/emulator:/Users/vojtad/Library/Android/sdk/tools:/Users/vojtad/Library/Android/sdk/tools/bin:/Users/vojtad/Library/Android/sdk/platform-tools",- "npm_config_allow_same_version": "",- "npm_package_devDependencies_jsdoc_export_default_interop": "^0.3.1",- "npm_package_contributors_1_email": "[email protected]",- "npm_config_https_proxy": "",- "npm_config_engine_strict": "",- "npm_config_description": "true",- "npm_config_userconfig": "/Users/vojtad/.npmrc",- "npm_config_init_module": "/Users/vojtad/.npm-init.js",- "npm_package_devDependencies_fpscanner": "^0.1.5",- "npm_package_keywords_4": "selenium",- "npm_config_cidr": "",- "npm_package_devDependencies_eslint_config_airbnb": "^17.1.0",- "npm_package_dependencies_puppeteer": "1.17.0",- "npm_package_keywords_5": "crawler",- "PWD": "/Users/vojtad/Documents/Apify/apify-js",- "npm_config_user": "",- "npm_config_node_version": "11.12.0",- "npm_package_bugs_url": "https://github.com/apifytech/apify-js/issues",- "npm_package_author_url": "https://apify.com",- "npm_package_keywords_6": "scraper",- "npm_lifecycle_event": "test",- "npm_package_devDependencies_chai": "^4.2.0",- "npm_config_save": "true",- "npm_config_ignore_prepublish": "",- "npm_config_editor": "vi",- "npm_config_auth_type": "legacy",- "npm_package_dependencies_selenium_webdriver": "^3.6.0",- "npm_package_repository_type": "git",- "npm_package_keywords_0": "apify",- "npm_package_name": "apify",- "LANG": "en_US.UTF-8",- "npm_config_tag": "latest",- "npm_config_script_shell": "",- "npm_package_devDependencies_eslint_plugin_import": "^2.14.0",- "npm_package_devDependencies__babel_preset_env": "^7.0.0",- "npm_package_dependencies_mime": "^2.3.1",- "npm_package_keywords_1": "headless",- "npm_config_progress": "true",- "npm_config_global": "",- "npm_package_devDependencies__babel_cli": "^7.0.0",- "npm_package_scripts_build": "rm -rf ./build && babel src --out-dir build --copy-files",- "npm_package_keywords_2": "chrome",- "npm_config_searchstaleness": "900",- "npm_config_optional": "true",- "npm_config_ham_it_up": "",- "npm_package_scripts_build_readme": "node ./tools/build_readme.js",- "npm_package_contributors_0_email": "[email protected]",- "npm_package_keywords_3": "puppeteer",- "XPC_FLAGS": "0x0",- "npm_config_save_prod": "",- "npm_config_force": "",- "npm_config_bin_links": "true",- "npm_config_searchopts": "",- "npm_package_engines_node": ">=8.14.0",- "npm_config_node_gyp": "/usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js",- "npm_config_depth": "Infinity",- "npm_package_main": "build/index.js",- "RBENV_SHELL": "bash",- "npm_config_sso_poll_frequency": "500",- "npm_config_rebuild_bundle": "true",- "npm_package_devDependencies_eslint_config_airbnb_base": "^13.1.0",- "npm_package_dependencies_ws": "^6.1.2",- "npm_package_version": "0.14.15",- "XPC_SERVICE_NAME": "0",- "npm_config_unicode": "true",- "npm_package_devDependencies_jsdoc": "^3.5.5",- "HOME": "/Users/vojtad",- "SHLVL": "8",- "npm_config_fetch_retry_maxtimeout": "60000",- "npm_package_devDependencies_nyc": "^14.0.0",- "npm_package_scripts_test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --require @babel/register --recursive --exit",- "APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL": "true",- "npm_config_tag_version_prefix": "v",- "npm_config_strict_ssl": "true",- "npm_config_sso_type": "oauth",- "npm_config_scripts_prepend_node_path": "warn-only",- "npm_config_save_prefix": "^",- "npm_config_loglevel": "notice",- "npm_config_ca": "",- "npm_package_dependencies_content_type": "^1.0.4",- "npm_config_save_exact": "",- "npm_config_group": "20",- "npm_config_fetch_retry_factor": "10",- "npm_config_dev": "",- "npm_package_scripts_build_docs": "npm run clean && npm run build && node ./website/tools/build_docs.js",- "npm_config_version": "",- "npm_config_prefer_offline": "",- "npm_config_cache_lock_stale": "60000",- "npm_package_devDependencies_eslint_plugin_promise": "^4.0.1",- "npm_config_otp": "",- "npm_config_cache_min": "10",- "npm_package_devDependencies_sinon_stub_promise": "^4.0.0",- "npm_config_searchexclude": "",- "npm_config_cache": "/Users/vojtad/.npm",- "npm_package_devDependencies__babel_register": "^7.0.0",- "npm_package_dependencies_fs_extra": "^7.0.1",- "LOGNAME": "vojtad",- "npm_lifecycle_script": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --require @babel/register --recursive --exit",- "npm_config_color": "true",- "npm_package_devDependencies_chai_as_promised": "^7.1.1",- "npm_config_proxy": "",- "npm_config_package_lock": "true",- "npm_package_author_email": "[email protected]",- "npm_config_package_lock_only": "",- "npm_config_save_optional": "",- "npm_package_dependencies_jquery": "^3.3.1",- "npm_config_ignore_scripts": "",- "npm_config_user_agent": "npm/6.7.0 node/v11.12.0 darwin x64",- "npm_config_cache_lock_wait": "10000",- "npm_package_contributors_2_email": "[email protected]",- "npm_config_production": "",- "DISPLAY": "/private/tmp/com.apple.launchd.nZo4KBo22U/org.macosforge.xquartz:0",- "npm_config_send_metrics": "",- "npm_config_save_bundle": "",- "npm_package_dependencies_express": "^4.16.4",- "npm_package_scripts_prepare": "npm run build",- "npm_package_files_0": "build",- "npm_config_umask": "0022",- "npm_config_node_options": "",- "npm_config_init_version": "1.0.0",- "npm_package_contributors_0_name": "Jan Curn",- "npm_config_init_author_name": "",- "npm_config_git": "git",- "npm_config_scope": "",- "npm_package_scripts_clean": "rm -rf build",- "SECURITYSESSIONID": "186a7",- "npm_config_unsafe_perm": "true",- "npm_config_tmp": "/var/folders/sw/01254g6n3nz7hbd9vx6slg840000gn/T",- "npm_config_onload_script": "",- "npm_node_execpath": "/usr/local/Cellar/node/11.12.0/bin/node",- "npm_config_link": "",- "npm_config_prefix": "/usr/local",- "npm_package_devDependencies_eslint_plugin_react": "^7.11.1",- "npm_package_contributors_1_name": "Marek Trunkat",- "_": "/Users/vojtad/Documents/Apify/apify-js/node_modules/.bin/nyc",- "BABEL_DISABLE_CACHE": "1",- "SPAWN_WRAP_SHIM_ROOT": "/Users/vojtad",- "NYC_CONFIG": "{\"_\":[\"mocha\"],\"silent\":false,\"s\":false,\"all\":false,\"a\":false,\"exclude-after-remap\":true,\"excludeAfterRemap\":true,\"exclude-node-modules\":true,\"excludeNodeModules\":true,\"eager\":false,\"cache\":true,\"c\":true,\"babel-cache\":false,\"babelCache\":false,\"es-modules\":true,\"esModules\":true,\"check-coverage\":false,\"checkCoverage\":false,\"source-map\":true,\"sourceMap\":true,\"per-file\":false,\"perFile\":false,\"produce-source-map\":false,\"produceSourceMap\":false,\"compact\":true,\"preserve-comments\":true,\"preserveComments\":true,\"instrument\":true,\"hook-require\":true,\"hookRequire\":true,\"hook-run-in-context\":false,\"hookRunInContext\":false,\"hook-run-in-this-context\":false,\"hookRunInThisContext\":false,\"show-process-tree\":false,\"showProcessTree\":false,\"clean\":true,\"skip-empty\":false,\"skipEmpty\":false,\"skip-full\":false,\"skipFull\":false,\"reporter\":[\"html\",\"text\"],\"r\":[\"html\",\"text\"],\"report-dir\":\"coverage\",\"reportDir\":\"coverage\",\"exclude\":[\"coverage/**\",\"packages/*/test/**\",\"test/**\",\"test{,-*}.js\",\"**/*{.,-}test.js\",\"**/__tests__/**\",\"**/{ava,babel,jest,nyc,rollup,webpack}.config.js\"],\"x\":[\"coverage/**\",\"packages/*/test/**\",\"test/**\",\"test{,-*}.js\",\"**/*{.,-}test.js\",\"**/__tests__/**\",\"**/{ava,babel,jest,nyc,rollup,webpack}.config.js\"],\"include\":[],\"n\":[],\"cwd\":\"/Users/vojtad/Documents/Apify/apify-js\",\"require\":[],\"i\":[],\"extension\":[],\"e\":[],\"branches\":0,\"functions\":0,\"lines\":90,\"statements\":0,\"nycrc-path\":\".nycrc\",\"nycrcPath\":\".nycrc\",\"temp-dir\":\"./.nyc_output\",\"t\":\"./.nyc_output\",\"tempDir\":\"./.nyc_output\",\"$0\":\"/Users/vojtad/Documents/Apify/apify-js/node_modules/.bin/nyc\",\"instrumenter\":\"./lib/instrumenters/istanbul\"}",- "NYC_CWD": "/Users/vojtad/Documents/Apify/apify-js",- "NYC_ROOT_ID": "f44d4862-6825-44f6-9e39-20380e5d6a44",- "NYC_INSTRUMENTER": "./lib/instrumenters/istanbul",- "NYC_PROCESS_ID": "f520af47-e030-4851-9c77-fd227fc6d50d",- "LOADED_MOCHA_OPTS": "true",- "SW_ORIG_50669-919223399a2c": "/usr/local/Cellar/node/11.12.0/bin/node",- "SPAWN_WRAP_SHIM_ROOT": "/Users/vojtad/.node-spawn-wrap-",- "APIFY_HEADLESS": "1",- "APIFY_MEMORY_MBYTES": "1234",- "APIFY_LOCAL_STORAGE_DIR": "/Users/vojtad/Documents/Apify/apify-js/apify_storage"- },- "userLimits": {- "core_file_size_blocks": {- "soft": 0,- "hard": "unlimited"- },- "data_seg_size_kbytes": {- "soft": "unlimited",- "hard": "unlimited"- },- "file_size_blocks": {- "soft": "unlimited",- "hard": "unlimited"- },- "max_locked_memory_bytes": {- "soft": "unlimited",- "hard": "unlimited"- },- "max_memory_size_kbytes": {- "soft": "unlimited",- "hard": "unlimited"- },- "open_files": {- "soft": 10240,- "hard": "unlimited"- },- "stack_size_bytes": {- "soft": 8388608,- "hard": 67104768- },- "cpu_time_seconds": {- "soft": "unlimited",- "hard": "unlimited"- },- "max_user_processes": {- "soft": 709,- "hard": 1064- },- "virtual_memory_kbytes": {- "soft": "unlimited",- "hard": "unlimited"- }- },- "sharedObjects": [- "/usr/local/Cellar/node/11.12.0/bin/node",- "/usr/local/opt/icu4c/lib/libicui18n.63.dylib",- "/usr/local/opt/icu4c/lib/libicuuc.63.dylib",- "/usr/local/opt/icu4c/lib/libicudata.63.1.dylib",- "/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation",- "/usr/lib/libSystem.B.dylib",- "/usr/lib/libc++.1.dylib",- "/usr/lib/system/libcache.dylib",- "/usr/lib/system/libcommonCrypto.dylib",- "/usr/lib/system/libcompiler_rt.dylib",- "/usr/lib/system/libcopyfile.dylib",- "/usr/lib/system/libcorecrypto.dylib",- "/usr/lib/system/libdispatch.dylib",- "/usr/lib/system/libdyld.dylib",- "/usr/lib/system/libkeymgr.dylib",- "/usr/lib/system/liblaunch.dylib",- "/usr/lib/system/libmacho.dylib",- "/usr/lib/system/libquarantine.dylib",- "/usr/lib/system/libremovefile.dylib",- "/usr/lib/system/libsystem_asl.dylib",- "/usr/lib/system/libsystem_blocks.dylib",- "/usr/lib/system/libsystem_c.dylib",- "/usr/lib/system/libsystem_configuration.dylib",- "/usr/lib/system/libsystem_coreservices.dylib",- "/usr/lib/system/libsystem_darwin.dylib",- "/usr/lib/system/libsystem_dnssd.dylib",- "/usr/lib/system/libsystem_info.dylib",- "/usr/lib/system/libsystem_m.dylib",- "/usr/lib/system/libsystem_malloc.dylib",- "/usr/lib/system/libsystem_network.dylib",- "/usr/lib/system/libsystem_networkextension.dylib",- "/usr/lib/system/libsystem_notify.dylib",- "/usr/lib/system/libsystem_sandbox.dylib",- "/usr/lib/system/libsystem_secinit.dylib",- "/usr/lib/system/libsystem_kernel.dylib",- "/usr/lib/system/libsystem_platform.dylib",- "/usr/lib/system/libsystem_pthread.dylib",- "/usr/lib/system/libsystem_symptoms.dylib",- "/usr/lib/system/libsystem_trace.dylib",- "/usr/lib/system/libunwind.dylib",- "/usr/lib/system/libxpc.dylib",- "/usr/lib/closure/libclosured.dylib",- "/usr/lib/libobjc.A.dylib",- "/usr/lib/libc++abi.dylib",- "/usr/lib/libDiagnosticMessagesClient.dylib",- "/usr/lib/libicucore.A.dylib",- "/usr/lib/libz.1.dylib",- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/ApplicationServices",- "/System/Library/Frameworks/CoreGraphics.framework/Versions/A/CoreGraphics",- "/System/Library/Frameworks/CoreText.framework/Versions/A/CoreText",- "/System/Library/Frameworks/ImageIO.framework/Versions/A/ImageIO",- "/System/Library/Frameworks/ColorSync.framework/Versions/A/ColorSync",- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATS.framework/Versions/A/ATS",- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ColorSyncLegacy.framework/Versions/A/ColorSyncLegacy",- "/System/Library/Frameworks/CoreServices.framework/Versions/A/CoreServices",- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/HIServices.framework/Versions/A/HIServices",- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/LangAnalysis.framework/Versions/A/LangAnalysis",- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/PrintCore.framework/Versions/A/PrintCore",- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/QD.framework/Versions/A/QD",- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/SpeechSynthesis.framework/Versions/A/SpeechSynthesis",- "/System/Library/PrivateFrameworks/SkyLight.framework/Versions/A/SkyLight",- "/System/Library/Frameworks/CFNetwork.framework/Versions/A/CFNetwork",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Accelerate",- "/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit",- "/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation",- "/System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration",- "/System/Library/Frameworks/CoreDisplay.framework/Versions/A/CoreDisplay",- "/System/Library/Frameworks/IOSurface.framework/Versions/A/IOSurface",- "/System/Library/Frameworks/Metal.framework/Versions/A/Metal",- "/System/Library/PrivateFrameworks/MultitouchSupport.framework/Versions/A/MultitouchSupport",- "/System/Library/Frameworks/Security.framework/Versions/A/Security",- "/System/Library/Frameworks/QuartzCore.framework/Versions/A/QuartzCore",- "/usr/lib/libbsm.0.dylib",- "/usr/lib/libauto.dylib",- "/System/Library/Frameworks/DiskArbitration.framework/Versions/A/DiskArbitration",- "/usr/lib/libarchive.2.dylib",- "/usr/lib/libxml2.2.dylib",- "/usr/lib/liblangid.dylib",- "/usr/lib/libCRFSuite.dylib",- "/usr/lib/libenergytrace.dylib",- "/usr/lib/system/libkxld.dylib",- "/usr/lib/libOpenScriptingUtil.dylib",- "/usr/lib/libcoretls.dylib",- "/usr/lib/libcoretls_cfhelpers.dylib",- "/usr/lib/libpam.2.dylib",- "/usr/lib/libsqlite3.dylib",- "/usr/lib/libxar.1.dylib",- "/usr/lib/libbz2.1.0.dylib",- "/usr/lib/liblzma.5.dylib",- "/usr/lib/libnetwork.dylib",- "/usr/lib/libapple_nghttp2.dylib",- "/usr/lib/libpcap.A.dylib",- "/usr/lib/libboringssl.dylib",- "/usr/lib/libusrtcp.dylib",- "/usr/lib/libapple_crypto.dylib",- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/FSEvents.framework/Versions/A/FSEvents",- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/CarbonCore.framework/Versions/A/CarbonCore",- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/Metadata.framework/Versions/A/Metadata",- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/OSServices.framework/Versions/A/OSServices",- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/SearchKit.framework/Versions/A/SearchKit",- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/AE.framework/Versions/A/AE",- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/LaunchServices.framework/Versions/A/LaunchServices",- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/DictionaryServices.framework/Versions/A/DictionaryServices",- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/SharedFileList.framework/Versions/A/SharedFileList",- "/System/Library/Frameworks/NetFS.framework/Versions/A/NetFS",- "/System/Library/PrivateFrameworks/NetAuth.framework/Versions/A/NetAuth",- "/System/Library/PrivateFrameworks/login.framework/Versions/A/Frameworks/loginsupport.framework/Versions/A/loginsupport",- "/System/Library/PrivateFrameworks/TCC.framework/Versions/A/TCC",- "/usr/lib/libmecabra.dylib",- "/System/Library/PrivateFrameworks/LinguisticData.framework/Versions/A/LinguisticData",- "/usr/lib/libmarisa.dylib",- "/System/Library/PrivateFrameworks/Lexicon.framework/Versions/A/Lexicon",- "/usr/lib/libChineseTokenizer.dylib",- "/usr/lib/libcmph.dylib",- "/usr/lib/libiconv.2.dylib",- "/System/Library/PrivateFrameworks/LanguageModeling.framework/Versions/A/LanguageModeling",- "/System/Library/Frameworks/CoreData.framework/Versions/A/CoreData",- "/System/Library/PrivateFrameworks/CoreEmoji.framework/Versions/A/CoreEmoji",- "/usr/lib/libcompression.dylib",- "/System/Library/Frameworks/OpenDirectory.framework/Versions/A/Frameworks/CFOpenDirectory.framework/Versions/A/CFOpenDirectory",- "/System/Library/PrivateFrameworks/APFS.framework/Versions/A/APFS",- "/usr/lib/libutil.dylib",- "/System/Library/Frameworks/ServiceManagement.framework/Versions/A/ServiceManagement",- "/System/Library/PrivateFrameworks/BackgroundTaskManagement.framework/Versions/A/BackgroundTaskManagement",- "/usr/lib/libxslt.1.dylib",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vImage.framework/Versions/A/vImage",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/vecLib",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libvDSP.dylib",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBNNS.dylib",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libQuadrature.dylib",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libvMisc.dylib",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libLAPACK.dylib",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBLAS.dylib",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libLinearAlgebra.dylib",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libSparse.dylib",- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libSparseBLAS.dylib",- "/System/Library/PrivateFrameworks/GPUWrangler.framework/Versions/A/GPUWrangler",- "/System/Library/PrivateFrameworks/IOAccelerator.framework/Versions/A/IOAccelerator",- "/System/Library/PrivateFrameworks/IOPresentment.framework/Versions/A/IOPresentment",- "/System/Library/PrivateFrameworks/DSExternalDisplay.framework/Versions/A/DSExternalDisplay",- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCoreFSCache.dylib",- "/System/Library/Frameworks/CoreImage.framework/Versions/A/CoreImage",- "/System/Library/Frameworks/CoreVideo.framework/Versions/A/CoreVideo",- "/System/Library/Frameworks/OpenGL.framework/Versions/A/OpenGL",- "/System/Library/PrivateFrameworks/GraphVisualizer.framework/Versions/A/GraphVisualizer",- "/System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/MetalPerformanceShaders",- "/usr/lib/libFosl_dynamic.dylib",- "/System/Library/PrivateFrameworks/FaceCore.framework/Versions/A/FaceCore",- "/System/Library/Frameworks/OpenCL.framework/Versions/A/OpenCL",- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATS.framework/Versions/A/Resources/libFontParser.dylib",- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATS.framework/Versions/A/Resources/libFontRegistry.dylib",- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libJPEG.dylib",- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libTIFF.dylib",- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libPng.dylib",- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libGIF.dylib",- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libJP2.dylib",- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libRadiance.dylib",- "/System/Library/PrivateFrameworks/AppleJPEG.framework/Versions/A/AppleJPEG",- "/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSCore.framework/Versions/A/MPSCore",- "/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSImage.framework/Versions/A/MPSImage",- "/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSNeuralNetwork.framework/Versions/A/MPSNeuralNetwork",- "/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSMatrix.framework/Versions/A/MPSMatrix",- "/System/Library/PrivateFrameworks/MetalTools.framework/Versions/A/MetalTools",- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGLU.dylib",- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGFXShared.dylib",- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGL.dylib",- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGLImage.dylib",- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCVMSPluginSupport.dylib",- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCoreVMClient.dylib",- "/usr/lib/libcups.2.dylib",- "/System/Library/Frameworks/Kerberos.framework/Versions/A/Kerberos",- "/System/Library/Frameworks/GSS.framework/Versions/A/GSS",- "/usr/lib/libresolv.9.dylib",- "/System/Library/PrivateFrameworks/Heimdal.framework/Versions/A/Heimdal",- "/usr/lib/libheimdal-asn1.dylib",- "/System/Library/Frameworks/OpenDirectory.framework/Versions/A/OpenDirectory",- "/System/Library/PrivateFrameworks/CommonAuth.framework/Versions/A/CommonAuth",- "/System/Library/Frameworks/SecurityFoundation.framework/Versions/A/SecurityFoundation",- "/System/Library/Frameworks/CoreAudio.framework/Versions/A/CoreAudio",- "/System/Library/Frameworks/AudioToolbox.framework/Versions/A/AudioToolbox",- "/System/Library/PrivateFrameworks/AppleSauce.framework/Versions/A/AppleSauce"- ]-}\ No newline at end of filediff --git a/src/utils.js b/src/utils.jsindex 5f12562c2..21a108a9d 100644--- a/src/utils.js+++ b/src/utils.js@@ -618,16 +618,21 @@ export const snakeCaseToCamelCase = (snakeCaseStr) => {/*** Scrolls to the bottom of a page, or until it times out.- * @param {Object} page- * @param {Number} [maxTimeout=0] How many seconds to scroll for. If 0, will scroll until bottom of page.- * @param {Number} [waitForDynamicContent=4] How many seconds to wait for no new content to load before exit.+ * @param {Object} options+ * @param {Object} options.page The page object returned from browser.newPage()+ * @param {Number} [options.timeoutSecs=0] How many seconds to scroll for. If 0, will scroll until bottom of page.+ * @param {Number} [options.waitForSecs=4] How many seconds to wait for no new content to load before exit.* @returns {Promise}* @memberOf utils*/-export const infiniteScroll = async function (page, maxTimeout = 0, waitForDynamicContent = 4) {+export const infiniteScroll = async function ({ page, timeoutSecs = 0, waitForSecs = 4 }) {+ checkParamOrThrow(page, 'page', 'Object');+ checkParamOrThrow(timeoutSecs, 'timeoutSecs', 'Number');+ checkParamOrThrow(waitForSecs, 'waitForSecs', 'Number');+let finished;- const MAX_TIMEOUT = maxTimeout;- const WAIT_FOR_DYNAMIC_CONTENT = waitForDynamicContent;+ const TIMEOUT_SECS = timeoutSecs;+ const WAIT_FOR_SECS = waitForSecs;const startTime = Date.now();const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];const resourcesStats = {@@ -636,7 +641,6 @@ export const infiniteScroll = async function (page, maxTimeout = 0, waitForDynammatchNumber: 0,};-page.on('request', (msg) => {if (maybeResourceTypesInfiniteScroll.includes(msg.resourceType())) {resourcesStats.newRequested++;@@ -647,7 +651,7 @@ export const infiniteScroll = async function (page, maxTimeout = 0, waitForDynam// console.log(resourcesStats)if (resourcesStats.oldRequested === resourcesStats.newRequested) {resourcesStats.matchNumber++;- if (resourcesStats.matchNumber >= WAIT_FOR_DYNAMIC_CONTENT) {+ if (resourcesStats.matchNumber >= WAIT_FOR_SECS) {clearInterval(checkFinished);finished = true;return;@@ -657,7 +661,7 @@ export const infiniteScroll = async function (page, maxTimeout = 0, waitForDynamresourcesStats.oldRequested = resourcesStats.newRequested;}// check if timeout has been reached- if (MAX_TIMEOUT !== 0 && (Date.now() - startTime) / 1000 > MAX_TIMEOUT) {+ if (TIMEOUT_SECS !== 0 && (Date.now() - startTime) / 1000 > TIMEOUT_SECS) {// console.log("Timeout limit reached, exiting infinite scroll.")clearInterval(checkFinished);finished = true;@@ -668,6 +672,7 @@ export const infiniteScroll = async function (page, maxTimeout = 0, waitForDynam/* global window document */const doScroll = async () => {+ /* istanbul ignore next */await page.evaluate(async () => {const delta = document.body.scrollHeight === 0 ? 10000 : document.body.scrollHeight; // in case scrollHeight fixed to 0window.scrollBy(0, delta);