Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Apply PR comments

Ondra Urbane3d2b064da0f
diff --git a/package.json b/package.json
index ec7a7b95c..905c6cf1d 100644
--- a/package.json
+++ b/package.json
@@ -57,6 +57,7 @@
"content-type": "^1.0.4",
"express": "^4.16.4",
"fs-extra": "^7.0.1",
+ "iltorb": "^2.4.2",
"jquery": "^3.3.1",
"mime": "^2.3.1",
"proxy-chain": "^0.2.5",
diff --git a/src/live_view/index.html b/src/live_view/index.html
index bdb4dabe2..208877e5f 100644
--- a/src/live_view/index.html
+++ b/src/live_view/index.html
@@ -27,7 +27,7 @@
<script src="https://cdnjs.cloudflare.com/ajax/libs/socket.io/2.2.0/socket.io.slim.js"></script>
<script src="https://code.jquery.com/jquery-1.11.1.min.js"></script>
<script src="./init-last-snapshot.js"></script>
- <link href="./prism.css" rel="stylesheet" />
+ <link href="https://cdnjs.cloudflare.com/ajax/libs/prism/1.16.0/themes/prism.min.css" rel="stylesheet" />
<script>
function showSnapshot(snapshot) {
if (!snapshot) return;
@@ -78,7 +78,7 @@
</head>
<body>
-<script src="./prism.js"></script>
+<script src="https://cdnjs.cloudflare.com/ajax/libs/prism/1.16.0/prism.js"></script>
<header id="header">
<button id="enableButton">Enable Live View</button>
diff --git a/src/live_view/live_view_server.js b/src/live_view/live_view_server.js
index 113958b20..6f3a344b9 100644
--- a/src/live_view/live_view_server.js
+++ b/src/live_view/live_view_server.js
@@ -18,7 +18,7 @@ const unlink = promisify(fs.unlink);
const ensureDir = promisify(fs.ensureDir);
/**
- * `LiveViewServer` enables serving of browser snapshots via web sockets. It includes it's own client
+ * `LiveViewServer` enables serving of browser snapshots via web sockets. It includes its own client
* that provides a simple frontend to viewing the captured snapshots. A snapshot consists of three
* pieces of information, the currently opened URL, the content of the page (HTML) and its screenshot.
*
@@ -30,13 +30,15 @@ const ensureDir = promisify(fs.ensureDir);
* NOTE: Screenshot taking in browser typically takes around 300ms. So having the `LiveViewServer`
* always serve snapshots will have a significant impact on performance.
*
- * When running on the Apify Platform and using {@link PuppeteerPool}, the `LiveViewServer` can be
+ * When using {@link PuppeteerPool}, the `LiveViewServer` can be
* easily used just by providing the `useLiveView = true` option to the {@link PuppeteerPool}.
+ * It can also be initiated via {@link PuppeteerCrawler} `puppeteerPoolOptions`.
+ *
* It will take snapshots of the first page of the latest browser. Taking snapshots of only a
* single page improves performance and stability dramatically in high concurrency situations.
*
- * When running locally, it is often best to use headful browser for debugging, since it provides
- * a better view into the browser, including DevTools.
+ * When running locally, it is often best to use a headful browser for debugging, since it provides
+ * a better view into the browser, including DevTools, but `LiveViewServer` works too.
*
* @param {Object} [options] All `LiveViewServer` parameters are passed
* via an options object with the following keys:
@@ -126,7 +128,7 @@ export default class LiveViewServer {
/**
* @return {boolean}
*/
- get isRunning() {
+ isRunning() {
return this._isRunning;
}
@@ -143,7 +145,7 @@ export default class LiveViewServer {
* @return {string}
*/
getScreenshotPath(screenshotIndex) {
- return path.join(this.screenshotDirectoryPath, `${screenshotIndex}`);
+ return path.join(this.screenshotDirectoryPath, `${screenshotIndex}.png`);
}
async _makeSnapshot(page) {
@@ -172,7 +174,12 @@ export default class LiveViewServer {
this.socketio.emit('snapshot', snapshot);
}
- _deleteScreenshot(screenshotIndex) { // eslint-disable-line class-methods-use-this
+ /**
+ * Initiates an async delete and does not wait for it to complete.
+ * @param screenshotIndex
+ * @ignore
+ */
+ _deleteScreenshot(screenshotIndex) {
unlink(this.getScreenshotPath(screenshotIndex))
.catch(err => log.exception(err, 'Cannot delete live view screenshot.'));
}
diff --git a/src/live_view/prism.css b/src/live_view/prism.css
deleted file mode 100644
index 742f8bbc1..000000000
--- a/src/live_view/prism.css
+++ /dev/null
@@ -1,140 +0,0 @@
-/* PrismJS 1.15.0
-https://prismjs.com/download.html#themes=prism&languages=markup */
-/**
- * prism.js default theme for JavaScript, CSS and HTML
- * Based on dabblet (http://dabblet.com)
- * @author Lea Verou
- */
-
-code[class*="language-"],
-pre[class*="language-"] {
- color: black;
- background: none;
- text-shadow: 0 1px white;
- font-family: Consolas, Monaco, 'Andale Mono', 'Ubuntu Mono', monospace;
- text-align: left;
- white-space: pre;
- word-spacing: normal;
- word-break: normal;
- word-wrap: normal;
- line-height: 1.5;
-
- -moz-tab-size: 4;
- -o-tab-size: 4;
- tab-size: 4;
-
- -webkit-hyphens: none;
- -moz-hyphens: none;
- -ms-hyphens: none;
- hyphens: none;
-}
-
-pre[class*="language-"]::-moz-selection, pre[class*="language-"] ::-moz-selection,
-code[class*="language-"]::-moz-selection, code[class*="language-"] ::-moz-selection {
- text-shadow: none;
- background: #b3d4fc;
-}
-
-pre[class*="language-"]::selection, pre[class*="language-"] ::selection,
-code[class*="language-"]::selection, code[class*="language-"] ::selection {
- text-shadow: none;
- background: #b3d4fc;
-}
-
-@media print {
- code[class*="language-"],
- pre[class*="language-"] {
- text-shadow: none;
- }
-}
-
-/* Code blocks */
-pre[class*="language-"] {
- padding: 1em;
- margin: .5em 0;
- overflow: auto;
-}
-
-:not(pre) > code[class*="language-"],
-pre[class*="language-"] {
- background: #f5f2f0;
-}
-
-/* Inline code */
-:not(pre) > code[class*="language-"] {
- padding: .1em;
- border-radius: .3em;
- white-space: normal;
-}
-
-.token.comment,
-.token.prolog,
-.token.doctype,
-.token.cdata {
- color: slategray;
-}
-
-.token.punctuation {
- color: #999;
-}
-
-.namespace {
- opacity: .7;
-}
-
-.token.property,
-.token.tag,
-.token.boolean,
-.token.number,
-.token.constant,
-.token.symbol,
-.token.deleted {
- color: #905;
-}
-
-.token.selector,
-.token.attr-name,
-.token.string,
-.token.char,
-.token.builtin,
-.token.inserted {
- color: #690;
-}
-
-.token.operator,
-.token.entity,
-.token.url,
-.language-css .token.string,
-.style .token.string {
- color: #9a6e3a;
- background: hsla(0, 0%, 100%, .5);
-}
-
-.token.atrule,
-.token.attr-value,
-.token.keyword {
- color: #07a;
-}
-
-.token.function,
-.token.class-name {
- color: #DD4A68;
-}
-
-.token.regex,
-.token.important,
-.token.variable {
- color: #e90;
-}
-
-.token.important,
-.token.bold {
- font-weight: bold;
-}
-.token.italic {
- font-style: italic;
-}
-
-.token.entity {
- cursor: help;
-}
diff --git a/src/live_view/prism.js b/src/live_view/prism.js
deleted file mode 100644
index 668bab7d7..000000000
--- a/src/live_view/prism.js
+++ /dev/null
@@ -1,5 +0,0 @@
-/* eslint-disable */
-/* PrismJS 1.15.0
-https://prismjs.com/download.html#themes=prism&languages=markup */
-var _self="undefined"!=typeof window?window:"undefined"!=typeof WorkerGlobalScope&&self instanceof WorkerGlobalScope?self:{},Prism=function(e){function t(e,t,a,n,r){this.type=e,this.content=t,this.alias=a,this.length=0|(n||"").length,this.greedy=!!r}var a=/\blang(?:uage)?-([\w-]+)\b/i,n=0,r={manual:e.Prism&&e.Prism.manual,disableWorkerMessageHandler:e.Prism&&e.Prism.disableWorkerMessageHandler,util:{encode:function(e){return e instanceof t?new t(e.type,r.util.encode(e.content),e.alias):"Array"===r.util.type(e)?e.map(r.util.encode):e.replace(/&/g,"&amp;").replace(/</g,"&lt;").replace(/\u00a0/g," ")},type:function(e){return Object.prototype.toString.call(e).slice(8,-1)},objId:function(e){return e.__id||Object.defineProperty(e,"__id",{value:++n}),e.__id},clone:function l(e,t){var a,n,i=r.util.type(e);switch(t=t||{},i){case"Object":if(n=r.util.objId(e),t[n])return t[n];a={},t[n]=a;for(var o in e)e.hasOwnProperty(o)&&(a[o]=l(e[o],t));return a;case"Array":return n=r.util.objId(e),t[n]?t[n]:(a=[],t[n]=a,e.forEach(function(e,n){a[n]=l(e,t)}),a);default:return e}}},languages:{extend:function(e,t){var a=r.util.clone(r.languages[e]);for(var n in t)a[n]=t[n];return a},insertBefore:function(e,t,a,n){n=n||r.languages;var i=n[e],l={};for(var o in i)if(i.hasOwnProperty(o)){if(o==t)for(var s in a)a.hasOwnProperty(s)&&(l[s]=a[s]);a.hasOwnProperty(o)||(l[o]=i[o])}var u=n[e];return n[e]=l,r.languages.DFS(r.languages,function(t,a){a===u&&t!=e&&(this[t]=l)}),l},DFS:function o(e,t,a,n){n=n||{};var i=r.util.objId;for(var l in e)if(e.hasOwnProperty(l)){t.call(e,l,e[l],a||l);var s=e[l],u=r.util.type(s);"Object"!==u||n[i(s)]?"Array"!==u||n[i(s)]||(n[i(s)]=!0,o(s,t,l,n)):(n[i(s)]=!0,o(s,t,null,n))}}},plugins:{},highlightAll:function(e,t){r.highlightAllUnder(document,e,t)},highlightAllUnder:function(e,t,a){var n={callback:a,selector:'code[class*="language-"], [class*="language-"] code, code[class*="lang-"], [class*="lang-"] code'};r.hooks.run("before-highlightall",n);for(var i,l=n.elements||e.querySelectorAll(n.selector),o=0;i=l[o++];)r.highlightElement(i,t===!0,n.callback)},highlightElement:function(t,n,i){for(var l,o,s=t;s&&!a.test(s.className);)s=s.parentNode;s&&(l=(s.className.match(a)||[,""])[1].toLowerCase(),o=r.languages[l]),t.className=t.className.replace(a,"").replace(/\s+/g," ")+" language-"+l,t.parentNode&&(s=t.parentNode,/pre/i.test(s.nodeName)&&(s.className=s.className.replace(a,"").replace(/\s+/g," ")+" language-"+l));var u=t.textContent,g={element:t,language:l,grammar:o,code:u},c=function(e){g.highlightedCode=e,r.hooks.run("before-insert",g),g.element.innerHTML=g.highlightedCode,r.hooks.run("after-highlight",g),r.hooks.run("complete",g),i&&i.call(g.element)};if(r.hooks.run("before-sanity-check",g),!g.code)return r.hooks.run("complete",g),void 0;if(r.hooks.run("before-highlight",g),!g.grammar)return c(r.util.encode(g.code)),void 0;if(n&&e.Worker){var h=new Worker(r.filename);h.onmessage=function(e){c(e.data)},h.postMessage(JSON.stringify({language:g.language,code:g.code,immediateClose:!0}))}else c(r.highlight(g.code,g.grammar,g.language))},highlight:function(e,a,n){var i={code:e,grammar:a,language:n};return r.hooks.run("before-tokenize",i),i.tokens=r.tokenize(i.code,i.grammar),r.hooks.run("after-tokenize",i),t.stringify(r.util.encode(i.tokens),i.language)},matchGrammar:function(e,a,n,i,l,o,s){for(var u in n)if(n.hasOwnProperty(u)&&n[u]){if(u==s)return;var g=n[u];g="Array"===r.util.type(g)?g:[g];for(var c=0;c<g.length;++c){var h=g[c],d=h.inside,f=!!h.lookbehind,p=!!h.greedy,m=0,y=h.alias;if(p&&!h.pattern.global){var v=h.pattern.toString().match(/[imuy]*$/)[0];h.pattern=RegExp(h.pattern.source,v+"g")}h=h.pattern||h;for(var k=i,b=l;k<a.length;b+=a[k].length,++k){var w=a[k];if(a.length>e.length)return;if(!(w instanceof t)){if(p&&k!=a.length-1){h.lastIndex=b;var A=h.exec(e);if(!A)break;for(var P=A.index+(f?A[1].length:0),O=A.index+A[0].length,x=k,N=b,j=a.length;j>x&&(O>N||!a[x].type&&!a[x-1].greedy);++x)N+=a[x].length,P>=N&&(++k,b=N);if(a[k]instanceof t)continue;S=x-k,w=e.slice(b,N),A.index-=b}else{h.lastIndex=0;var A=h.exec(w),S=1}if(A){f&&(m=A[1]?A[1].length:0);var P=A.index+m,A=A[0].slice(m),O=P+A.length,E=w.slice(0,P),_=w.slice(O),C=[k,S];E&&(++k,b+=E.length,C.push(E));var M=new t(u,d?r.tokenize(A,d):A,y,A,p);if(C.push(M),_&&C.push(_),Array.prototype.splice.apply(a,C),1!=S&&r.matchGrammar(e,a,n,k,b,!0,u),o)break}else if(o)break}}}}},tokenize:function(e,t){var a=[e],n=t.rest;if(n){for(var i in n)t[i]=n[i];delete t.rest}return r.matchGrammar(e,a,t,0,0,!1),a},hooks:{all:{},add:function(e,t){var a=r.hooks.all;a[e]=a[e]||[],a[e].push(t)},run:function(e,t){var a=r.hooks.all[e];if(a&&a.length)for(var n,i=0;n=a[i++];)n(t)}},Token:t};if(e.Prism=r,t.stringify=function(e,a,n){if("string"==typeof e)return e;if("Array"===r.util.type(e))return e.map(function(n){return t.stringify(n,a,e)}).join("");var i={type:e.type,content:t.stringify(e.content,a,n),tag:"span",classes:["token",e.type],attributes:{},language:a,parent:n};if(e.alias){var l="Array"===r.util.type(e.alias)?e.alias:[e.alias];Array.prototype.push.apply(i.classes,l)}r.hooks.run("wrap",i);var o=Object.keys(i.attributes).map(function(e){return e+'="'+(i.attributes[e]||"").replace(/"/g,"&quot;")+'"'}).join(" ");return"<"+i.tag+' class="'+i.classes.join(" ")+'"'+(o?" "+o:"")+">"+i.content+"</"+i.tag+">"},!e.document)return e.addEventListener?(r.disableWorkerMessageHandler||e.addEventListener("message",function(t){var a=JSON.parse(t.data),n=a.language,i=a.code,l=a.immediateClose;e.postMessage(r.highlight(i,r.languages[n],n)),l&&e.close()},!1),r):r;var i=document.currentScript||[].slice.call(document.getElementsByTagName("script")).pop();return i&&(r.filename=i.src,r.manual||i.hasAttribute("data-manual")||("loading"!==document.readyState?window.requestAnimationFrame?window.requestAnimationFrame(r.highlightAll):window.setTimeout(r.highlightAll,16):document.addEventListener("DOMContentLoaded",r.highlightAll))),r}(_self);"undefined"!=typeof module&&module.exports&&(module.exports=Prism),"undefined"!=typeof global&&(global.Prism=Prism);
-Prism.languages.markup={comment:/<!--[\s\S]*?-->/,prolog:/<\?[\s\S]+?\?>/,doctype:/<!DOCTYPE[\s\S]+?>/i,cdata:/<!\[CDATA\[[\s\S]*?]]>/i,tag:{pattern:/<\/?(?!\d)[^\s>\/=$<%]+(?:\s(?:\s*[^\s>\/=]+(?:\s*=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+(?=[\s>]))|(?=[\s\/>])))+)?\s*\/?>/i,greedy:!0,inside:{tag:{pattern:/^<\/?[^\s>\/]+/i,inside:{punctuation:/^<\/?/,namespace:/^[^\s>\/:]+:/}},"attr-value":{pattern:/=\s*(?:"[^"]*"|'[^']*'|[^\s'">=]+)/i,inside:{punctuation:[/^=/,{pattern:/^(\s*)["']|["']$/,lookbehind:!0}]}},punctuation:/\/?>/,"attr-name":{pattern:/[^\s>\/]+/,inside:{namespace:/^[^\s>\/:]+:/}}}},entity:/&#?[\da-z]{1,8};/i},Prism.languages.markup.tag.inside["attr-value"].inside.entity=Prism.languages.markup.entity,Prism.hooks.add("wrap",function(a){"entity"===a.type&&(a.attributes.title=a.content.replace(/&amp;/,"&"))}),Prism.languages.xml=Prism.languages.extend("markup",{}),Prism.languages.html=Prism.languages.markup,Prism.languages.mathml=Prism.languages.markup,Prism.languages.svg=Prism.languages.markup;
diff --git a/src/puppeteer_pool.js b/src/puppeteer_pool.js
index a5e27d40d..488a35d5f 100644
--- a/src/puppeteer_pool.js
+++ b/src/puppeteer_pool.js
@@ -206,10 +206,7 @@ class PuppeteerPool {
this.killInstanceAfterMillis = killInstanceAfterMillis || killInstanceAfterSecs * 1000;
this.recycledDiskCacheDirs = recycleDiskCache ? new LinkedList() : null;
this.proxyUrls = proxyUrls ? _.shuffle(proxyUrls) : null;
- if (useLiveView && !isAtHome()) {
- log.warning('LiveView can only be used on the Apify Platform. Use headful mode for debugging locally.');
- }
- this.liveViewServer = useLiveView && isAtHome() ? new LiveViewServer() : null;
+ this.liveViewServer = useLiveView ? new LiveViewServer() : null;
this.launchPuppeteerFunction = async () => {
// Do not modify passed launchPuppeteerOptions!
const opts = _.clone(launchPuppeteerOptions) || {};
@@ -471,7 +468,7 @@ class PuppeteerPool {
* @return {Promise<Page>}
*/
async newPage() {
- if (this.liveViewServer && !this.liveViewServer.isRunning) await this.liveViewServer.start();
+ if (this.liveViewServer && !this.liveViewServer.isRunning()) await this.liveViewServer.start();
let idlePage;
// We don't need to check whether options.reusePages is true,
// because if it's false, the array will be empty and the loop will never start.
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 41b9d7436..0e351bfbe 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -475,7 +475,7 @@ describe('KeyValueStore', () => {
describe('getInput', async () => {
it('should work', async () => {
- process.env[ENV_VARS.LOCAL_STORAGE_DIR] = 'something';
+ process.env[ENV_VARS.LOCAL_STORAGE_DIR] = LOCAL_STORAGE_DIR;
const defaultStore = await Apify.openKeyValueStore();
// Uses default value.
diff --git a/test/live_view/live_view_server.js b/test/live_view/live_view_server.js
index 055c293d0..7a847d785 100644
--- a/test/live_view/live_view_server.js
+++ b/test/live_view/live_view_server.js
@@ -47,9 +47,9 @@ describe('LiveViewServer', () => {
it('should start and stop', async () => {
await lvs.start();
- expect(lvs.isRunning).to.be.eql(true);
+ expect(lvs.isRunning()).to.be.eql(true);
await lvs.stop();
- expect(lvs.isRunning).to.be.eql(false);
+ expect(lvs.isRunning()).to.be.eql(false);
});
it('should connect over websocket', async () => {
@@ -116,7 +116,7 @@ describe('LiveViewServer', () => {
reject(new Error('Files were not deleted in 2000ms.'));
}, 2000);
});
- files.forEach((f, idx) => expect(f).to.be.eql(`${idx + 3}`));
+ files.forEach((f, idx) => expect(f).to.be.eql(`${idx + 3}.png`));
});
});
});