Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

options object

vdrmotae5d65fad18c2
diff --git a/report.20190619.113804.50671.001.json b/report.20190619.113804.50671.001.json
deleted file mode 100644
index 848ecaca7..000000000
--- a/report.20190619.113804.50671.001.json
+++ /dev/null
@@ -1,671 +0,0 @@
-
-{
- "header": {
- "event": "Allocation failed - JavaScript heap out of memory",
- "trigger": "FatalError",
- "filename": "report.20190619.113804.50671.001.json",
- "dumpEventTime": "2019-06-19T11:38:04Z",
- "dumpEventTimeStamp": "1560937084495",
- "processId": 50671,
- "commandLine": [
- "/usr/local/Cellar/node/11.12.0/bin/node",
- "/Users/vojtad/.node-spawn-wrap-50669-919223399a2c/node",
- "/Users/vojtad/Documents/Apify/apify-js/node_modules/mocha/bin/_mocha",
- "--timeout",
- "60000",
- "--require",
- "@babel/register",
- "--recursive",
- "--exit"
- ],
- "nodejsVersion": "v11.12.0",
- "wordSize": 64,
- "arch": "x64",
- "platform": "darwin",
- "componentVersions": {
- "node": "11.12.0",
- "v8": "7.0.276.38-node.18",
- "uv": "1.26.0",
- "zlib": "1.2.11",
- "brotli": "1.0.7",
- "ares": "1.15.0",
- "modules": "67",
- "nghttp2": "1.34.0",
- "napi": "4",
- "llhttp": "1.1.1",
- "http_parser": "2.8.0",
- "openssl": "1.1.1b",
- "cldr": "34.0",
- "icu": "63.1",
- "tz": "2018e",
- "unicode": "11.0"
- },
- "release": {
- "name": "node",
- "headersUrl": "https://nodejs.org/download/release/v11.12.0/node-v11.12.0-headers.tar.gz",
- "sourceUrl": "https://nodejs.org/download/release/v11.12.0/node-v11.12.0.tar.gz"
- },
- "osName": "Darwin",
- "osRelease": "17.7.0",
- "osVersion": "Darwin Kernel Version 17.7.0: Thu Jun 21 22:53:14 PDT 2018; root:xnu-4570.71.2~1/RELEASE_X86_64",
- "osMachine": "x86_64",
- "host": "Vojtas-MacBook-Air.local"
- },
- "javascriptStack": {
- "message": "No stack.",
- "stack": [
- "Unavailable."
- ]
- },
- "nativeStack": [
- {
- "pc": "0x0000000100127384",
- "symbol": "report::TriggerNodeReport(v8::Isolate*, node::Environment*, char const*, char const*, std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, v8::Local<v8::String>) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x0000000100064c2a",
- "symbol": "node::OnFatalError(char const*, char const*) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x000000010017e40f",
- "symbol": "v8::Utils::ReportOOMFailure(v8::internal::Isolate*, char const*, bool) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x000000010017e3b0",
- "symbol": "v8::internal::V8::FatalProcessOutOfMemory(v8::internal::Isolate*, char const*, bool) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x0000000100440714",
- "symbol": "v8::internal::Heap::UpdateSurvivalStatistics(int) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x0000000100442131",
- "symbol": "v8::internal::Heap::CheckIneffectiveMarkCompact(unsigned long, double) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x000000010043faa9",
- "symbol": "v8::internal::Heap::PerformGarbageCollection(v8::internal::GarbageCollector, v8::GCCallbackFlags) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x000000010043e89f",
- "symbol": "v8::internal::Heap::CollectGarbage(v8::internal::AllocationSpace, v8::internal::GarbageCollectionReason, v8::GCCallbackFlags) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x000000010044676d",
- "symbol": "v8::internal::Heap::AllocateRawWithLightRetry(int, v8::internal::AllocationSpace, v8::internal::AllocationAlignment) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x00000001004467bc",
- "symbol": "v8::internal::Heap::AllocateRawWithRetryOrFail(int, v8::internal::AllocationSpace, v8::internal::AllocationAlignment) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x0000000100426b83",
- "symbol": "v8::internal::Factory::NewFillerObject(int, bool, v8::internal::AllocationSpace) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x0000000100603f9e",
- "symbol": "v8::internal::Runtime_AllocateInNewSpace(int, v8::internal::Object**, v8::internal::Isolate*) [/usr/local/Cellar/node/11.12.0/bin/node]"
- },
- {
- "pc": "0x00003c15fcc4fc7d",
- "symbol": ""
- }
- ],
- "javascriptHeap": {
- "totalMemory": 1473159168,
- "totalCommittedMemory": 1470533744,
- "usedMemory": 1441614376,
- "availableMemory": 55362560,
- "memoryLimit": 1526909922,
- "heapSpaces": {
- "read_only_space": {
- "memorySize": 524288,
- "committedMemory": 42224,
- "capacity": 515584,
- "used": 33520,
- "available": 482064
- },
- "new_space": {
- "memorySize": 2097152,
- "committedMemory": 1119392,
- "capacity": 1031168,
- "used": 53568,
- "available": 977600
- },
- "old_space": {
- "memorySize": 1460342784,
- "committedMemory": 1459907712,
- "capacity": 1434753560,
- "used": 1434409448,
- "available": 344112
- },
- "code_space": {
- "memorySize": 2621440,
- "committedMemory": 2108096,
- "capacity": 1921888,
- "used": 1921888,
- "available": 0
- },
- "map_space": {
- "memorySize": 2633728,
- "committedMemory": 2416544,
- "capacity": 1623680,
- "used": 1623680,
- "available": 0
- },
- "large_object_space": {
- "memorySize": 4939776,
- "committedMemory": 4939776,
- "capacity": 57131056,
- "used": 3572272,
- "available": 53558784
- },
- "new_large_object_space": {
- "memorySize": 0,
- "committedMemory": 0,
- "capacity": 0,
- "used": 0,
- "available": 0
- }
- }
- },
- "resourceUsage": {
- "userCpuSeconds": 115.251,
- "kernelCpuSeconds": 5.14185,
- "cpuConsumptionPercent": 120.393,
- "maxRss": 1554782355456,
- "pageFaults": {
- "IORequired": 71,
- "IONotRequired": 804958
- },
- "fsActivity": {
- "reads": 0,
- "writes": 0
- }
- },
- "libuv": [
- ],
- "environmentVariables": {
- "npm_config_save_dev": "",
- "npm_config_legacy_bundling": "",
- "npm_config_dry_run": "",
- "npm_package_devDependencies_chai_subset": "^1.6.0",
- "npm_package_dependencies_request": "^2.88.0",
- "npm_package_dependencies__apify_ps_tree": "^1.1.3",
- "npm_config_viewer": "man",
- "npm_config_only": "",
- "npm_config_commit_hooks": "true",
- "npm_config_browser": "",
- "npm_package_gitHead": "769598b4f8dd62ca2be0e0e580636b2a05e0eeb9",
- "npm_config_also": "",
- "npm_package_dependencies_apify_client": "^0.5.19",
- "npm_config_sign_git_commit": "",
- "npm_config_rollback": "true",
- "TERM_PROGRAM": "vscode",
- "NODE": "/usr/local/Cellar/node/11.12.0/bin/node",
- "npm_config_usage": "",
- "npm_config_audit": "true",
- "npm_package_devDependencies_socket_io_client": "^2.2.0",
- "npm_package_devDependencies_fpcollect": "^1.0.4",
- "npm_package_dependencies_cheerio": "^1.0.0-rc.2",
- "INIT_CWD": "/Users/vojtad/Documents/Apify/apify-js",
- "npm_package_devDependencies_sinon": "^7.1.1",
- "npm_package_devDependencies__types_puppeteer": "^1.12.4",
- "npm_package_dependencies_proxy_chain": "^0.2.5",
- "npm_package_dependencies_iltorb": "^2.4.2",
- "npm_package_homepage": "https://sdk.apify.com/",
- "ANDROID_HOME": "/Users/vojtad/Library/Android/sdk",
- "npm_config_globalignorefile": "/usr/local/etc/npmignore",
- "npm_package_devDependencies_mocha": "^5.2.0",
- "TERM": "xterm-256color",
- "SHELL": "/bin/bash",
- "npm_config_shell": "/bin/bash",
- "npm_config_maxsockets": "50",
- "npm_config_init_author_url": "",
- "npm_package_devDependencies_portastic": "^1.0.1",
- "npm_package_devDependencies_basic_auth_parser": "^0.0.2",
- "npm_package_dependencies_htmlparser2": "^3.10.1",
- "npm_config_shrinkwrap": "true",
- "npm_config_parseable": "",
- "npm_config_metrics_registry": "https://registry.npmjs.org/",
- "npm_package_dependencies_xregexp": "^4.2.4",
- "TMPDIR": "/var/folders/sw/01254g6n3nz7hbd9vx6slg840000gn/T/",
- "npm_config_timing": "",
- "npm_config_init_license": "ISC",
- "npm_package_dependencies_socket_io": "^2.2.0",
- "npm_package_scripts_lint": "npm run build && ./node_modules/.bin/eslint ./src ./test",
- "Apple_PubSub_Socket_Render": "/private/tmp/com.apple.launchd.EPJ9mKsnYj/Render",
- "npm_config_if_present": "",
- "npm_package_dependencies_rimraf": "^2.6.2",
- "TERM_PROGRAM_VERSION": "1.34.0",
- "npm_config_sign_git_tag": "",
- "npm_config_init_author_email": "",
- "npm_config_cache_max": "Infinity",
- "npm_package_optionalDependencies_selenium_webdriver": "^3.6.0",
- "npm_package_scripts_prepublishOnly": "(test $RUNNING_FROM_SCRIPT || (echo \"You must use publish.sh instead of 'npm publish' directly!\"; exit 1)) && npm test && npm run lint",
- "npm_config_preid": "",
- "npm_config_long": "",
- "npm_config_local_address": "",
- "npm_config_git_tag_version": "true",
- "npm_config_cert": "",
- "TERM_SESSION_ID": "D0D76223-185D-43EB-B4FA-41E0940819D9",
- "npm_config_registry": "https://registry.npmjs.org/",
- "npm_config_noproxy": "",
- "npm_config_fetch_retries": "2",
- "npm_package_repository_url": "git+https://github.com/apifytech/apify-js.git",
- "npm_config_versions": "",
- "npm_config_message": "%s",
- "npm_config_key": "",
- "npm_package_readmeFilename": "README.md",
- "npm_package_devDependencies_why_is_node_running": "^2.0.3",
- "npm_package_devDependencies_eslint_plugin_jsx_a11y": "^6.1.2",
- "npm_package_description": "The scalable web crawling and scraping library for JavaScript. Enables development of data extraction and web automation jobs (not only) with headless Chrome and Puppeteer.",
- "USER": "vojtad",
- "npm_package_license": "Apache-2.0",
- "npm_config_globalconfig": "/usr/local/etc/npmrc",
- "npm_package_devDependencies_proxy": "^0.2.4",
- "npm_package_dependencies_request_promise_native": "^1.0.5",
- "npm_package_dependencies_apify_shared": "^0.1.51",
- "npm_config_prefer_online": "",
- "npm_config_logs_max": "10",
- "npm_config_always_auth": "",
- "npm_package_devDependencies__babel_core": "^7.0.0",
- "npm_package_contributors_2_name": "Ondra Urban",
- "SSH_AUTH_SOCK": "/private/tmp/com.apple.launchd.Gh7whWq4Uh/Listeners",
- "npm_package_devDependencies_eslint": "^5.9.0",
- "__CF_USER_TEXT_ENCODING": "0x1F5:0x0:0x0",
- "npm_execpath": "/usr/local/lib/node_modules/npm/bin/npm-cli.js",
- "npm_config_global_style": "",
- "npm_config_cache_lock_retries": "10",
- "npm_config_update_notifier": "true",
- "npm_config_cafile": "",
- "npm_package_optionalDependencies_puppeteer": "1.17.0",
- "npm_package_author_name": "Apify",
- "npm_config_heading": "npm",
- "npm_config_audit_level": "low",
- "npm_config_searchlimit": "20",
- "npm_config_read_only": "",
- "npm_config_offline": "",
- "npm_config_fetch_retry_mintimeout": "10000",
- "npm_config_json": "",
- "npm_config_access": "",
- "npm_config_argv": "{\"remain\":[],\"cooked\":[\"run\",\"test\"],\"original\":[\"run\",\"test\"]}",
- "npm_package_devDependencies__types_socket_io": "^2.1.2",
- "npm_package_dependencies_underscore": "^1.9.1",
- "npm_package_scripts_lint_fix": "eslint ./src ./test --ext .js,.jsx --fix",
- "PATH": "/usr/local/lib/node_modules/npm/node_modules/npm-lifecycle/node-gyp-bin:/Users/vojtad/Documents/Apify/apify-js/node_modules/.bin:/Library/Frameworks/Python.framework/Versions/3.6/bin:/Users/vojtad/.rbenv/shims:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/opt/X11/bin:/Library/Frameworks/Python.framework/Versions/3.6/bin:/Users/vojtad/.rbenv/shims:/Users/vojtad/Library/Android/sdk/emulator:/Users/vojtad/Library/Android/sdk/tools:/Users/vojtad/Library/Android/sdk/tools/bin:/Users/vojtad/Library/Android/sdk/platform-tools:/Users/vojtad/Library/Android/sdk/emulator:/Users/vojtad/Library/Android/sdk/tools:/Users/vojtad/Library/Android/sdk/tools/bin:/Users/vojtad/Library/Android/sdk/platform-tools",
- "npm_config_allow_same_version": "",
- "npm_package_devDependencies_jsdoc_export_default_interop": "^0.3.1",
- "npm_package_contributors_1_email": "[email protected]",
- "npm_config_https_proxy": "",
- "npm_config_engine_strict": "",
- "npm_config_description": "true",
- "npm_config_userconfig": "/Users/vojtad/.npmrc",
- "npm_config_init_module": "/Users/vojtad/.npm-init.js",
- "npm_package_devDependencies_fpscanner": "^0.1.5",
- "npm_package_keywords_4": "selenium",
- "npm_config_cidr": "",
- "npm_package_devDependencies_eslint_config_airbnb": "^17.1.0",
- "npm_package_dependencies_puppeteer": "1.17.0",
- "npm_package_keywords_5": "crawler",
- "PWD": "/Users/vojtad/Documents/Apify/apify-js",
- "npm_config_user": "",
- "npm_config_node_version": "11.12.0",
- "npm_package_bugs_url": "https://github.com/apifytech/apify-js/issues",
- "npm_package_author_url": "https://apify.com",
- "npm_package_keywords_6": "scraper",
- "npm_lifecycle_event": "test",
- "npm_package_devDependencies_chai": "^4.2.0",
- "npm_config_save": "true",
- "npm_config_ignore_prepublish": "",
- "npm_config_editor": "vi",
- "npm_config_auth_type": "legacy",
- "npm_package_dependencies_selenium_webdriver": "^3.6.0",
- "npm_package_repository_type": "git",
- "npm_package_keywords_0": "apify",
- "npm_package_name": "apify",
- "LANG": "en_US.UTF-8",
- "npm_config_tag": "latest",
- "npm_config_script_shell": "",
- "npm_package_devDependencies_eslint_plugin_import": "^2.14.0",
- "npm_package_devDependencies__babel_preset_env": "^7.0.0",
- "npm_package_dependencies_mime": "^2.3.1",
- "npm_package_keywords_1": "headless",
- "npm_config_progress": "true",
- "npm_config_global": "",
- "npm_package_devDependencies__babel_cli": "^7.0.0",
- "npm_package_scripts_build": "rm -rf ./build && babel src --out-dir build --copy-files",
- "npm_package_keywords_2": "chrome",
- "npm_config_searchstaleness": "900",
- "npm_config_optional": "true",
- "npm_config_ham_it_up": "",
- "npm_package_scripts_build_readme": "node ./tools/build_readme.js",
- "npm_package_contributors_0_email": "[email protected]",
- "npm_package_keywords_3": "puppeteer",
- "XPC_FLAGS": "0x0",
- "npm_config_save_prod": "",
- "npm_config_force": "",
- "npm_config_bin_links": "true",
- "npm_config_searchopts": "",
- "npm_package_engines_node": ">=8.14.0",
- "npm_config_node_gyp": "/usr/local/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js",
- "npm_config_depth": "Infinity",
- "npm_package_main": "build/index.js",
- "RBENV_SHELL": "bash",
- "npm_config_sso_poll_frequency": "500",
- "npm_config_rebuild_bundle": "true",
- "npm_package_devDependencies_eslint_config_airbnb_base": "^13.1.0",
- "npm_package_dependencies_ws": "^6.1.2",
- "npm_package_version": "0.14.15",
- "XPC_SERVICE_NAME": "0",
- "npm_config_unicode": "true",
- "npm_package_devDependencies_jsdoc": "^3.5.5",
- "HOME": "/Users/vojtad",
- "SHLVL": "8",
- "npm_config_fetch_retry_maxtimeout": "60000",
- "npm_package_devDependencies_nyc": "^14.0.0",
- "npm_package_scripts_test": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --require @babel/register --recursive --exit",
- "APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL": "true",
- "npm_config_tag_version_prefix": "v",
- "npm_config_strict_ssl": "true",
- "npm_config_sso_type": "oauth",
- "npm_config_scripts_prepend_node_path": "warn-only",
- "npm_config_save_prefix": "^",
- "npm_config_loglevel": "notice",
- "npm_config_ca": "",
- "npm_package_dependencies_content_type": "^1.0.4",
- "npm_config_save_exact": "",
- "npm_config_group": "20",
- "npm_config_fetch_retry_factor": "10",
- "npm_config_dev": "",
- "npm_package_scripts_build_docs": "npm run clean && npm run build && node ./website/tools/build_docs.js",
- "npm_config_version": "",
- "npm_config_prefer_offline": "",
- "npm_config_cache_lock_stale": "60000",
- "npm_package_devDependencies_eslint_plugin_promise": "^4.0.1",
- "npm_config_otp": "",
- "npm_config_cache_min": "10",
- "npm_package_devDependencies_sinon_stub_promise": "^4.0.0",
- "npm_config_searchexclude": "",
- "npm_config_cache": "/Users/vojtad/.npm",
- "npm_package_devDependencies__babel_register": "^7.0.0",
- "npm_package_dependencies_fs_extra": "^7.0.1",
- "LOGNAME": "vojtad",
- "npm_lifecycle_script": "npm run build && nyc --reporter=html --reporter=text mocha --timeout 60000 --require @babel/register --recursive --exit",
- "npm_config_color": "true",
- "npm_package_devDependencies_chai_as_promised": "^7.1.1",
- "npm_config_proxy": "",
- "npm_config_package_lock": "true",
- "npm_package_author_email": "[email protected]",
- "npm_config_package_lock_only": "",
- "npm_config_save_optional": "",
- "npm_package_dependencies_jquery": "^3.3.1",
- "npm_config_ignore_scripts": "",
- "npm_config_user_agent": "npm/6.7.0 node/v11.12.0 darwin x64",
- "npm_config_cache_lock_wait": "10000",
- "npm_package_contributors_2_email": "[email protected]",
- "npm_config_production": "",
- "DISPLAY": "/private/tmp/com.apple.launchd.nZo4KBo22U/org.macosforge.xquartz:0",
- "npm_config_send_metrics": "",
- "npm_config_save_bundle": "",
- "npm_package_dependencies_express": "^4.16.4",
- "npm_package_scripts_prepare": "npm run build",
- "npm_package_files_0": "build",
- "npm_config_umask": "0022",
- "npm_config_node_options": "",
- "npm_config_init_version": "1.0.0",
- "npm_package_contributors_0_name": "Jan Curn",
- "npm_config_init_author_name": "",
- "npm_config_git": "git",
- "npm_config_scope": "",
- "npm_package_scripts_clean": "rm -rf build",
- "SECURITYSESSIONID": "186a7",
- "npm_config_unsafe_perm": "true",
- "npm_config_tmp": "/var/folders/sw/01254g6n3nz7hbd9vx6slg840000gn/T",
- "npm_config_onload_script": "",
- "npm_node_execpath": "/usr/local/Cellar/node/11.12.0/bin/node",
- "npm_config_link": "",
- "npm_config_prefix": "/usr/local",
- "npm_package_devDependencies_eslint_plugin_react": "^7.11.1",
- "npm_package_contributors_1_name": "Marek Trunkat",
- "_": "/Users/vojtad/Documents/Apify/apify-js/node_modules/.bin/nyc",
- "BABEL_DISABLE_CACHE": "1",
- "SPAWN_WRAP_SHIM_ROOT": "/Users/vojtad",
- "NYC_CONFIG": "{\"_\":[\"mocha\"],\"silent\":false,\"s\":false,\"all\":false,\"a\":false,\"exclude-after-remap\":true,\"excludeAfterRemap\":true,\"exclude-node-modules\":true,\"excludeNodeModules\":true,\"eager\":false,\"cache\":true,\"c\":true,\"babel-cache\":false,\"babelCache\":false,\"es-modules\":true,\"esModules\":true,\"check-coverage\":false,\"checkCoverage\":false,\"source-map\":true,\"sourceMap\":true,\"per-file\":false,\"perFile\":false,\"produce-source-map\":false,\"produceSourceMap\":false,\"compact\":true,\"preserve-comments\":true,\"preserveComments\":true,\"instrument\":true,\"hook-require\":true,\"hookRequire\":true,\"hook-run-in-context\":false,\"hookRunInContext\":false,\"hook-run-in-this-context\":false,\"hookRunInThisContext\":false,\"show-process-tree\":false,\"showProcessTree\":false,\"clean\":true,\"skip-empty\":false,\"skipEmpty\":false,\"skip-full\":false,\"skipFull\":false,\"reporter\":[\"html\",\"text\"],\"r\":[\"html\",\"text\"],\"report-dir\":\"coverage\",\"reportDir\":\"coverage\",\"exclude\":[\"coverage/**\",\"packages/*/test/**\",\"test/**\",\"test{,-*}.js\",\"**/*{.,-}test.js\",\"**/__tests__/**\",\"**/{ava,babel,jest,nyc,rollup,webpack}.config.js\"],\"x\":[\"coverage/**\",\"packages/*/test/**\",\"test/**\",\"test{,-*}.js\",\"**/*{.,-}test.js\",\"**/__tests__/**\",\"**/{ava,babel,jest,nyc,rollup,webpack}.config.js\"],\"include\":[],\"n\":[],\"cwd\":\"/Users/vojtad/Documents/Apify/apify-js\",\"require\":[],\"i\":[],\"extension\":[],\"e\":[],\"branches\":0,\"functions\":0,\"lines\":90,\"statements\":0,\"nycrc-path\":\".nycrc\",\"nycrcPath\":\".nycrc\",\"temp-dir\":\"./.nyc_output\",\"t\":\"./.nyc_output\",\"tempDir\":\"./.nyc_output\",\"$0\":\"/Users/vojtad/Documents/Apify/apify-js/node_modules/.bin/nyc\",\"instrumenter\":\"./lib/instrumenters/istanbul\"}",
- "NYC_CWD": "/Users/vojtad/Documents/Apify/apify-js",
- "NYC_ROOT_ID": "f44d4862-6825-44f6-9e39-20380e5d6a44",
- "NYC_INSTRUMENTER": "./lib/instrumenters/istanbul",
- "NYC_PROCESS_ID": "f520af47-e030-4851-9c77-fd227fc6d50d",
- "LOADED_MOCHA_OPTS": "true",
- "SW_ORIG_50669-919223399a2c": "/usr/local/Cellar/node/11.12.0/bin/node",
- "SPAWN_WRAP_SHIM_ROOT": "/Users/vojtad/.node-spawn-wrap-",
- "APIFY_HEADLESS": "1",
- "APIFY_MEMORY_MBYTES": "1234",
- "APIFY_LOCAL_STORAGE_DIR": "/Users/vojtad/Documents/Apify/apify-js/apify_storage"
- },
- "userLimits": {
- "core_file_size_blocks": {
- "soft": 0,
- "hard": "unlimited"
- },
- "data_seg_size_kbytes": {
- "soft": "unlimited",
- "hard": "unlimited"
- },
- "file_size_blocks": {
- "soft": "unlimited",
- "hard": "unlimited"
- },
- "max_locked_memory_bytes": {
- "soft": "unlimited",
- "hard": "unlimited"
- },
- "max_memory_size_kbytes": {
- "soft": "unlimited",
- "hard": "unlimited"
- },
- "open_files": {
- "soft": 10240,
- "hard": "unlimited"
- },
- "stack_size_bytes": {
- "soft": 8388608,
- "hard": 67104768
- },
- "cpu_time_seconds": {
- "soft": "unlimited",
- "hard": "unlimited"
- },
- "max_user_processes": {
- "soft": 709,
- "hard": 1064
- },
- "virtual_memory_kbytes": {
- "soft": "unlimited",
- "hard": "unlimited"
- }
- },
- "sharedObjects": [
- "/usr/local/Cellar/node/11.12.0/bin/node",
- "/usr/local/opt/icu4c/lib/libicui18n.63.dylib",
- "/usr/local/opt/icu4c/lib/libicuuc.63.dylib",
- "/usr/local/opt/icu4c/lib/libicudata.63.1.dylib",
- "/System/Library/Frameworks/CoreFoundation.framework/Versions/A/CoreFoundation",
- "/usr/lib/libSystem.B.dylib",
- "/usr/lib/libc++.1.dylib",
- "/usr/lib/system/libcache.dylib",
- "/usr/lib/system/libcommonCrypto.dylib",
- "/usr/lib/system/libcompiler_rt.dylib",
- "/usr/lib/system/libcopyfile.dylib",
- "/usr/lib/system/libcorecrypto.dylib",
- "/usr/lib/system/libdispatch.dylib",
- "/usr/lib/system/libdyld.dylib",
- "/usr/lib/system/libkeymgr.dylib",
- "/usr/lib/system/liblaunch.dylib",
- "/usr/lib/system/libmacho.dylib",
- "/usr/lib/system/libquarantine.dylib",
- "/usr/lib/system/libremovefile.dylib",
- "/usr/lib/system/libsystem_asl.dylib",
- "/usr/lib/system/libsystem_blocks.dylib",
- "/usr/lib/system/libsystem_c.dylib",
- "/usr/lib/system/libsystem_configuration.dylib",
- "/usr/lib/system/libsystem_coreservices.dylib",
- "/usr/lib/system/libsystem_darwin.dylib",
- "/usr/lib/system/libsystem_dnssd.dylib",
- "/usr/lib/system/libsystem_info.dylib",
- "/usr/lib/system/libsystem_m.dylib",
- "/usr/lib/system/libsystem_malloc.dylib",
- "/usr/lib/system/libsystem_network.dylib",
- "/usr/lib/system/libsystem_networkextension.dylib",
- "/usr/lib/system/libsystem_notify.dylib",
- "/usr/lib/system/libsystem_sandbox.dylib",
- "/usr/lib/system/libsystem_secinit.dylib",
- "/usr/lib/system/libsystem_kernel.dylib",
- "/usr/lib/system/libsystem_platform.dylib",
- "/usr/lib/system/libsystem_pthread.dylib",
- "/usr/lib/system/libsystem_symptoms.dylib",
- "/usr/lib/system/libsystem_trace.dylib",
- "/usr/lib/system/libunwind.dylib",
- "/usr/lib/system/libxpc.dylib",
- "/usr/lib/closure/libclosured.dylib",
- "/usr/lib/libobjc.A.dylib",
- "/usr/lib/libc++abi.dylib",
- "/usr/lib/libDiagnosticMessagesClient.dylib",
- "/usr/lib/libicucore.A.dylib",
- "/usr/lib/libz.1.dylib",
- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/ApplicationServices",
- "/System/Library/Frameworks/CoreGraphics.framework/Versions/A/CoreGraphics",
- "/System/Library/Frameworks/CoreText.framework/Versions/A/CoreText",
- "/System/Library/Frameworks/ImageIO.framework/Versions/A/ImageIO",
- "/System/Library/Frameworks/ColorSync.framework/Versions/A/ColorSync",
- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATS.framework/Versions/A/ATS",
- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ColorSyncLegacy.framework/Versions/A/ColorSyncLegacy",
- "/System/Library/Frameworks/CoreServices.framework/Versions/A/CoreServices",
- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/HIServices.framework/Versions/A/HIServices",
- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/LangAnalysis.framework/Versions/A/LangAnalysis",
- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/PrintCore.framework/Versions/A/PrintCore",
- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/QD.framework/Versions/A/QD",
- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/SpeechSynthesis.framework/Versions/A/SpeechSynthesis",
- "/System/Library/PrivateFrameworks/SkyLight.framework/Versions/A/SkyLight",
- "/System/Library/Frameworks/CFNetwork.framework/Versions/A/CFNetwork",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Accelerate",
- "/System/Library/Frameworks/IOKit.framework/Versions/A/IOKit",
- "/System/Library/Frameworks/Foundation.framework/Versions/C/Foundation",
- "/System/Library/Frameworks/SystemConfiguration.framework/Versions/A/SystemConfiguration",
- "/System/Library/Frameworks/CoreDisplay.framework/Versions/A/CoreDisplay",
- "/System/Library/Frameworks/IOSurface.framework/Versions/A/IOSurface",
- "/System/Library/Frameworks/Metal.framework/Versions/A/Metal",
- "/System/Library/PrivateFrameworks/MultitouchSupport.framework/Versions/A/MultitouchSupport",
- "/System/Library/Frameworks/Security.framework/Versions/A/Security",
- "/System/Library/Frameworks/QuartzCore.framework/Versions/A/QuartzCore",
- "/usr/lib/libbsm.0.dylib",
- "/usr/lib/libauto.dylib",
- "/System/Library/Frameworks/DiskArbitration.framework/Versions/A/DiskArbitration",
- "/usr/lib/libarchive.2.dylib",
- "/usr/lib/libxml2.2.dylib",
- "/usr/lib/liblangid.dylib",
- "/usr/lib/libCRFSuite.dylib",
- "/usr/lib/libenergytrace.dylib",
- "/usr/lib/system/libkxld.dylib",
- "/usr/lib/libOpenScriptingUtil.dylib",
- "/usr/lib/libcoretls.dylib",
- "/usr/lib/libcoretls_cfhelpers.dylib",
- "/usr/lib/libpam.2.dylib",
- "/usr/lib/libsqlite3.dylib",
- "/usr/lib/libxar.1.dylib",
- "/usr/lib/libbz2.1.0.dylib",
- "/usr/lib/liblzma.5.dylib",
- "/usr/lib/libnetwork.dylib",
- "/usr/lib/libapple_nghttp2.dylib",
- "/usr/lib/libpcap.A.dylib",
- "/usr/lib/libboringssl.dylib",
- "/usr/lib/libusrtcp.dylib",
- "/usr/lib/libapple_crypto.dylib",
- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/FSEvents.framework/Versions/A/FSEvents",
- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/CarbonCore.framework/Versions/A/CarbonCore",
- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/Metadata.framework/Versions/A/Metadata",
- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/OSServices.framework/Versions/A/OSServices",
- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/SearchKit.framework/Versions/A/SearchKit",
- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/AE.framework/Versions/A/AE",
- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/LaunchServices.framework/Versions/A/LaunchServices",
- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/DictionaryServices.framework/Versions/A/DictionaryServices",
- "/System/Library/Frameworks/CoreServices.framework/Versions/A/Frameworks/SharedFileList.framework/Versions/A/SharedFileList",
- "/System/Library/Frameworks/NetFS.framework/Versions/A/NetFS",
- "/System/Library/PrivateFrameworks/NetAuth.framework/Versions/A/NetAuth",
- "/System/Library/PrivateFrameworks/login.framework/Versions/A/Frameworks/loginsupport.framework/Versions/A/loginsupport",
- "/System/Library/PrivateFrameworks/TCC.framework/Versions/A/TCC",
- "/usr/lib/libmecabra.dylib",
- "/System/Library/PrivateFrameworks/LinguisticData.framework/Versions/A/LinguisticData",
- "/usr/lib/libmarisa.dylib",
- "/System/Library/PrivateFrameworks/Lexicon.framework/Versions/A/Lexicon",
- "/usr/lib/libChineseTokenizer.dylib",
- "/usr/lib/libcmph.dylib",
- "/usr/lib/libiconv.2.dylib",
- "/System/Library/PrivateFrameworks/LanguageModeling.framework/Versions/A/LanguageModeling",
- "/System/Library/Frameworks/CoreData.framework/Versions/A/CoreData",
- "/System/Library/PrivateFrameworks/CoreEmoji.framework/Versions/A/CoreEmoji",
- "/usr/lib/libcompression.dylib",
- "/System/Library/Frameworks/OpenDirectory.framework/Versions/A/Frameworks/CFOpenDirectory.framework/Versions/A/CFOpenDirectory",
- "/System/Library/PrivateFrameworks/APFS.framework/Versions/A/APFS",
- "/usr/lib/libutil.dylib",
- "/System/Library/Frameworks/ServiceManagement.framework/Versions/A/ServiceManagement",
- "/System/Library/PrivateFrameworks/BackgroundTaskManagement.framework/Versions/A/BackgroundTaskManagement",
- "/usr/lib/libxslt.1.dylib",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vImage.framework/Versions/A/vImage",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/vecLib",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libvDSP.dylib",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBNNS.dylib",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libQuadrature.dylib",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libvMisc.dylib",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libLAPACK.dylib",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libBLAS.dylib",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libLinearAlgebra.dylib",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libSparse.dylib",
- "/System/Library/Frameworks/Accelerate.framework/Versions/A/Frameworks/vecLib.framework/Versions/A/libSparseBLAS.dylib",
- "/System/Library/PrivateFrameworks/GPUWrangler.framework/Versions/A/GPUWrangler",
- "/System/Library/PrivateFrameworks/IOAccelerator.framework/Versions/A/IOAccelerator",
- "/System/Library/PrivateFrameworks/IOPresentment.framework/Versions/A/IOPresentment",
- "/System/Library/PrivateFrameworks/DSExternalDisplay.framework/Versions/A/DSExternalDisplay",
- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCoreFSCache.dylib",
- "/System/Library/Frameworks/CoreImage.framework/Versions/A/CoreImage",
- "/System/Library/Frameworks/CoreVideo.framework/Versions/A/CoreVideo",
- "/System/Library/Frameworks/OpenGL.framework/Versions/A/OpenGL",
- "/System/Library/PrivateFrameworks/GraphVisualizer.framework/Versions/A/GraphVisualizer",
- "/System/Library/Frameworks/MetalPerformanceShaders.framework/Versions/A/MetalPerformanceShaders",
- "/usr/lib/libFosl_dynamic.dylib",
- "/System/Library/PrivateFrameworks/FaceCore.framework/Versions/A/FaceCore",
- "/System/Library/Frameworks/OpenCL.framework/Versions/A/OpenCL",
- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATS.framework/Versions/A/Resources/libFontParser.dylib",
- "/System/Library/Frameworks/ApplicationServices.framework/Versions/A/Frameworks/ATS.framework/Versions/A/Resources/libFontRegistry.dylib",
- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libJPEG.dylib",
- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libTIFF.dylib",
- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libPng.dylib",
- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libGIF.dylib",
- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libJP2.dylib",
- "/System/Library/Frameworks/ImageIO.framework/Versions/A/Resources/libRadiance.dylib",
- "/System/Library/PrivateFrameworks/AppleJPEG.framework/Versions/A/AppleJPEG",
- "/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSCore.framework/Versions/A/MPSCore",
- "/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSImage.framework/Versions/A/MPSImage",
- "/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSNeuralNetwork.framework/Versions/A/MPSNeuralNetwork",
- "/System/Library/Frameworks/MetalPerformanceShaders.framework/Frameworks/MPSMatrix.framework/Versions/A/MPSMatrix",
- "/System/Library/PrivateFrameworks/MetalTools.framework/Versions/A/MetalTools",
- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGLU.dylib",
- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGFXShared.dylib",
- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGL.dylib",
- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libGLImage.dylib",
- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCVMSPluginSupport.dylib",
- "/System/Library/Frameworks/OpenGL.framework/Versions/A/Libraries/libCoreVMClient.dylib",
- "/usr/lib/libcups.2.dylib",
- "/System/Library/Frameworks/Kerberos.framework/Versions/A/Kerberos",
- "/System/Library/Frameworks/GSS.framework/Versions/A/GSS",
- "/usr/lib/libresolv.9.dylib",
- "/System/Library/PrivateFrameworks/Heimdal.framework/Versions/A/Heimdal",
- "/usr/lib/libheimdal-asn1.dylib",
- "/System/Library/Frameworks/OpenDirectory.framework/Versions/A/OpenDirectory",
- "/System/Library/PrivateFrameworks/CommonAuth.framework/Versions/A/CommonAuth",
- "/System/Library/Frameworks/SecurityFoundation.framework/Versions/A/SecurityFoundation",
- "/System/Library/Frameworks/CoreAudio.framework/Versions/A/CoreAudio",
- "/System/Library/Frameworks/AudioToolbox.framework/Versions/A/AudioToolbox",
- "/System/Library/PrivateFrameworks/AppleSauce.framework/Versions/A/AppleSauce"
- ]
-}
\ No newline at end of file
diff --git a/src/utils.js b/src/utils.js
index 5f12562c2..21a108a9d 100644
--- a/src/utils.js
+++ b/src/utils.js
@@ -618,16 +618,21 @@ export const snakeCaseToCamelCase = (snakeCaseStr) => {
/**
* Scrolls to the bottom of a page, or until it times out.
- * @param {Object} page
- * @param {Number} [maxTimeout=0] How many seconds to scroll for. If 0, will scroll until bottom of page.
- * @param {Number} [waitForDynamicContent=4] How many seconds to wait for no new content to load before exit.
+ * @param {Object} options
+ * @param {Object} options.page The page object returned from browser.newPage()
+ * @param {Number} [options.timeoutSecs=0] How many seconds to scroll for. If 0, will scroll until bottom of page.
+ * @param {Number} [options.waitForSecs=4] How many seconds to wait for no new content to load before exit.
* @returns {Promise}
* @memberOf utils
*/
-export const infiniteScroll = async function (page, maxTimeout = 0, waitForDynamicContent = 4) {
+export const infiniteScroll = async function ({ page, timeoutSecs = 0, waitForSecs = 4 }) {
+ checkParamOrThrow(page, 'page', 'Object');
+ checkParamOrThrow(timeoutSecs, 'timeoutSecs', 'Number');
+ checkParamOrThrow(waitForSecs, 'waitForSecs', 'Number');
+
let finished;
- const MAX_TIMEOUT = maxTimeout;
- const WAIT_FOR_DYNAMIC_CONTENT = waitForDynamicContent;
+ const TIMEOUT_SECS = timeoutSecs;
+ const WAIT_FOR_SECS = waitForSecs;
const startTime = Date.now();
const maybeResourceTypesInfiniteScroll = ['xhr', 'fetch', 'websocket', 'other'];
const resourcesStats = {
@@ -636,7 +641,6 @@ export const infiniteScroll = async function (page, maxTimeout = 0, waitForDynam
matchNumber: 0,
};
-
page.on('request', (msg) => {
if (maybeResourceTypesInfiniteScroll.includes(msg.resourceType())) {
resourcesStats.newRequested++;
@@ -647,7 +651,7 @@ export const infiniteScroll = async function (page, maxTimeout = 0, waitForDynam
// console.log(resourcesStats)
if (resourcesStats.oldRequested === resourcesStats.newRequested) {
resourcesStats.matchNumber++;
- if (resourcesStats.matchNumber >= WAIT_FOR_DYNAMIC_CONTENT) {
+ if (resourcesStats.matchNumber >= WAIT_FOR_SECS) {
clearInterval(checkFinished);
finished = true;
return;
@@ -657,7 +661,7 @@ export const infiniteScroll = async function (page, maxTimeout = 0, waitForDynam
resourcesStats.oldRequested = resourcesStats.newRequested;
}
// check if timeout has been reached
- if (MAX_TIMEOUT !== 0 && (Date.now() - startTime) / 1000 > MAX_TIMEOUT) {
+ if (TIMEOUT_SECS !== 0 && (Date.now() - startTime) / 1000 > TIMEOUT_SECS) {
// console.log("Timeout limit reached, exiting infinite scroll.")
clearInterval(checkFinished);
finished = true;
@@ -668,6 +672,7 @@ export const infiniteScroll = async function (page, maxTimeout = 0, waitForDynam
/* global window document */
const doScroll = async () => {
+ /* istanbul ignore next */
await page.evaluate(async () => {
const delta = document.body.scrollHeight === 0 ? 10000 : document.body.scrollHeight; // in case scrollHeight fixed to 0
window.scrollBy(0, delta);