Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Add compileScript() function

Ondra Urbanf99b51c39199
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 628cd48bf..8b40e0795 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -1,4 +1,5 @@
import fs from 'fs';
+import vm from 'vm';
import Promise from 'bluebird';
import _ from 'underscore';
import log from 'apify-shared/log';
@@ -190,6 +191,44 @@ const blockResources = async (page, resourceTypes = ['stylesheet', 'font', 'imag
});
};
+/**
+ * Compiles a Puppeteer script into an async function that may be executed at any time
+ * by providing it with the following object:
+ * ```
+ * {
+ * page: Puppeteer.Page,
+ * request: Apify.Request,
+ * }
+ * ```
+ * The function is compiled by using the scriptString parameter as the function's body,
+ * so any limitations to function bodies apply. Return value of the function is the return
+ * value of the function body = scriptString parameter.
+ *
+ * As a security measure, no globals such as 'process' or 'require' are accessible
+ * from within the function body. Note that the function does not provide a safe
+ * sandbox and even though globals are not easily accessible, malicious code may
+ * still execute in the main process via prototype manipulation. Therefore you
+ * should only use this function to execute sanitized or safe code.
+ *
+ * @param {String} scriptString
+ * @return {Function} async ({ page, request }) => { scriptString }
+ */
+const compileScript = (scriptString) => {
+ const funcString = `async ({ page, request }) => {${scriptString}}`;
+
+ let func;
+ try {
+ func = vm.runInNewContext(funcString, Object.create(null)); // "Secure" the context by removing prototypes.
+ } catch (err) {
+ log.exception(err, 'Cannot compile script!');
+ throw err;
+ }
+
+ if (!_.isFunction(func)) throw new Error('Compilation result is not a function!'); // This should not happen...
+
+ return func;
+};
+
/**
* A namespace that contains various Puppeteer utilities.
@@ -217,4 +256,5 @@ export const puppeteerUtils = {
enqueueRequestsFromClickableElements,
enqueueLinks,
blockResources,
+ compileScript,
};
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index 56c8b063d..9e8b96d27 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -223,4 +223,31 @@ describe('Apify.utils.puppeteer', () => {
browser.close();
}
});
+
+ it('compileScript() works', async () => {
+ const { compileScript } = Apify.utils.puppeteer;
+ const scriptStringGood = 'await page.goto("about:blank"); return await page.content();';
+ const scriptStringBad = 'for const while';
+ const script = compileScript(scriptStringGood);
+
+ expect(script).to.be.a('function');
+ expect(script.toString()).to.be.eql(`async ({ page, request }) => {${scriptStringGood}}`);
+
+ try {
+ compileScript(scriptStringBad);
+ throw new Error('Should fail.');
+ } catch (err) {
+ // TODO figure out why the err.message comes out empty in the logs.
+ expect(err.message).to.include('Unexpected token const');
+ }
+ const browser = await Apify.launchPuppeteer({ headless: true });
+ try {
+ const page = await browser.newPage();
+ const content = await script({ page });
+ expect(content).to.be.a('string');
+ expect(content).to.be.eql('<html><head></head><body></body></html>');
+ } finally {
+ browser.close();
+ }
+ });
});