Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

Make survival optional

Ondra Urban4125182f536b
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.js
index 367cb6371..909fa80aa 100644
--- a/src/puppeteer_utils.js
+++ b/src/puppeteer_utils.js
@@ -9,8 +9,8 @@ import { RequestQueue, RequestQueueLocal } from './request_queue';
import Request from './request';
import { enqueueLinks } from './enqueue_links';
-const jqueryPath = require.resolve('jquery');
-const underscorePath = require.resolve('underscore');
+const jqueryPath = require.resolve('jquery/dist/jquery.min');
+const underscorePath = require.resolve('underscore/underscore-min');
const readFilePromised = util.promisify(fs.readFile);
/**
@@ -62,16 +62,23 @@ const hideWebDriver = async (page) => {
* @param {Page} page
* Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.
* @param {String} filePath File path
+ * @param {Object} [options]
+ * @param {boolean} [options.surviveNavigations]
+ * Enables the injected script to survive page navigations and reloads without need to be re-injected manually.
+ * This does not mean, however, that internal state will be preserved. Just that it will be automatically
+ * re-injected on each navigation before any other scripts get the chance to execute.
* @return {Promise}
* @memberOf puppeteer
*/
-const injectFile = async (page, filePath) => {
+const injectFile = async (page, filePath, options = {}) => {
checkParamOrThrow(page, 'page', 'Object');
checkParamOrThrow(filePath, 'filePath', 'String');
+ checkParamOrThrow(options, 'options', 'Object');
const contents = await readFilePromised(filePath, 'utf8');
+ if (options.surviveNavigations) await page.evaluateOnNewDocument(contents);
- return page.evaluateOnNewDocument(contents);
+ return page.evaluate(contents);
};
/**
@@ -104,7 +111,7 @@ const injectJQuery = (page) => {
checkParamOrThrow(page, 'page', 'Object');
// TODO: For better performance we could use minimized version of the script
- return injectFile(page, jqueryPath);
+ return injectFile(page, jqueryPath, { surviveNavigations: true });
};
/**
@@ -130,7 +137,7 @@ const injectUnderscore = (page) => {
checkParamOrThrow(page, 'page', 'Object');
// TODO: For better performance we could use minimized version of the script
- return injectFile(page, underscorePath);
+ return injectFile(page, underscorePath, { surviveNavigations: true });
};
/**
diff --git a/test/data/inject_file.js b/test/data/inject_file.txt
similarity index 100%
rename from test/data/inject_file.js
rename to test/data/inject_file.txt
diff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.js
index cf8e17893..d4c628c49 100644
--- a/test/puppeteer_utils.js
+++ b/test/puppeteer_utils.js
@@ -20,18 +20,40 @@ describe('Apify.utils.puppeteer', () => {
});
it('injectFile()', async () => {
+ /* eslint-disable no-shadow */
const browser = await Apify.launchPuppeteer({ headless: true });
- try {
+ const survive = async (browser) => {
+ // Survive navigations
const page = await browser.newPage();
- await Apify.utils.puppeteer.injectFile(page, path.join(__dirname, 'data', 'inject_file.js'));
- // let result = await page.evaluate(() => window.injectedVariable);
- // expect(result).to.be.eql(42);
+ let result = await page.evaluate(() => window.injectedVariable === 42);
+ expect(result).to.be.eql(false);
+ await Apify.utils.puppeteer.injectFile(page, path.join(__dirname, 'data', 'inject_file.txt'), { surviveNavigations: true });
+ result = await page.evaluate(() => window.injectedVariable);
+ expect(result).to.be.eql(42);
await page.goto('about:chrome');
- let result = await page.evaluate(() => window.injectedVariable);
+ result = await page.evaluate(() => window.injectedVariable);
expect(result).to.be.eql(42);
await page.goto('https://www.example.com');
result = await page.evaluate(() => window.injectedVariable);
expect(result).to.be.eql(42);
+ };
+ const remove = async (browser) => {
+ // Remove with navigations
+ const page = await browser.newPage();
+ let result = await page.evaluate(() => window.injectedVariable === 42);
+ expect(result).to.be.eql(false);
+ await page.goto('about:chrome');
+ result = await page.evaluate(() => window.injectedVariable === 42);
+ expect(result).to.be.eql(false);
+ await Apify.utils.puppeteer.injectFile(page, path.join(__dirname, 'data', 'inject_file.txt'));
+ result = await page.evaluate(() => window.injectedVariable);
+ expect(result).to.be.eql(42);
+ await page.goto('https://www.example.com');
+ result = await page.evaluate(() => window.injectedVariable === 42);
+ expect(result).to.be.eql(false);
+ };
+ try {
+ await Promise.all([survive(browser), remove(browser)]);
} finally {
browser.close();
}