Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.
TypeScript apifyautomationcrawlercrawlingheadlessheadless-chromejavascriptnodejsnpmplaywrightpuppeteerscraperscrapingtypescriptweb-crawlerweb-crawlingweb-scraping
Make survival optional
Ondra Urban4125182f536b
diff --git a/src/puppeteer_utils.js b/src/puppeteer_utils.jsindex 367cb6371..909fa80aa 100644--- a/src/puppeteer_utils.js+++ b/src/puppeteer_utils.js@@ -9,8 +9,8 @@ import { RequestQueue, RequestQueueLocal } from './request_queue';import Request from './request';import { enqueueLinks } from './enqueue_links';-const jqueryPath = require.resolve('jquery');-const underscorePath = require.resolve('underscore');+const jqueryPath = require.resolve('jquery/dist/jquery.min');+const underscorePath = require.resolve('underscore/underscore-min');const readFilePromised = util.promisify(fs.readFile);/**@@ -62,16 +62,23 @@ const hideWebDriver = async (page) => {* @param {Page} page* Puppeteer <a href="https://pptr.dev/#?product=Puppeteer&show=api-class-page" target="_blank"><code>Page</code></a> object.* @param {String} filePath File path+ * @param {Object} [options]+ * @param {boolean} [options.surviveNavigations]+ * Enables the injected script to survive page navigations and reloads without need to be re-injected manually.+ * This does not mean, however, that internal state will be preserved. Just that it will be automatically+ * re-injected on each navigation before any other scripts get the chance to execute.* @return {Promise}* @memberOf puppeteer*/-const injectFile = async (page, filePath) => {+const injectFile = async (page, filePath, options = {}) => {checkParamOrThrow(page, 'page', 'Object');checkParamOrThrow(filePath, 'filePath', 'String');+ checkParamOrThrow(options, 'options', 'Object');const contents = await readFilePromised(filePath, 'utf8');+ if (options.surviveNavigations) await page.evaluateOnNewDocument(contents);- return page.evaluateOnNewDocument(contents);+ return page.evaluate(contents);};/**@@ -104,7 +111,7 @@ const injectJQuery = (page) => {checkParamOrThrow(page, 'page', 'Object');// TODO: For better performance we could use minimized version of the script- return injectFile(page, jqueryPath);+ return injectFile(page, jqueryPath, { surviveNavigations: true });};/**@@ -130,7 +137,7 @@ const injectUnderscore = (page) => {checkParamOrThrow(page, 'page', 'Object');// TODO: For better performance we could use minimized version of the script- return injectFile(page, underscorePath);+ return injectFile(page, underscorePath, { surviveNavigations: true });};/**diff --git a/test/data/inject_file.js b/test/data/inject_file.txtsimilarity index 100%rename from test/data/inject_file.jsrename to test/data/inject_file.txtdiff --git a/test/puppeteer_utils.js b/test/puppeteer_utils.jsindex cf8e17893..d4c628c49 100644--- a/test/puppeteer_utils.js+++ b/test/puppeteer_utils.js@@ -20,18 +20,40 @@ describe('Apify.utils.puppeteer', () => {});it('injectFile()', async () => {+ /* eslint-disable no-shadow */const browser = await Apify.launchPuppeteer({ headless: true });- try {+ const survive = async (browser) => {+ // Survive navigationsconst page = await browser.newPage();- await Apify.utils.puppeteer.injectFile(page, path.join(__dirname, 'data', 'inject_file.js'));- // let result = await page.evaluate(() => window.injectedVariable);- // expect(result).to.be.eql(42);+ let result = await page.evaluate(() => window.injectedVariable === 42);+ expect(result).to.be.eql(false);+ await Apify.utils.puppeteer.injectFile(page, path.join(__dirname, 'data', 'inject_file.txt'), { surviveNavigations: true });+ result = await page.evaluate(() => window.injectedVariable);+ expect(result).to.be.eql(42);await page.goto('about:chrome');- let result = await page.evaluate(() => window.injectedVariable);+ result = await page.evaluate(() => window.injectedVariable);expect(result).to.be.eql(42);await page.goto('https://www.example.com');result = await page.evaluate(() => window.injectedVariable);expect(result).to.be.eql(42);+ };+ const remove = async (browser) => {+ // Remove with navigations+ const page = await browser.newPage();+ let result = await page.evaluate(() => window.injectedVariable === 42);+ expect(result).to.be.eql(false);+ await page.goto('about:chrome');+ result = await page.evaluate(() => window.injectedVariable === 42);+ expect(result).to.be.eql(false);+ await Apify.utils.puppeteer.injectFile(page, path.join(__dirname, 'data', 'inject_file.txt'));+ result = await page.evaluate(() => window.injectedVariable);+ expect(result).to.be.eql(42);+ await page.goto('https://www.example.com');+ result = await page.evaluate(() => window.injectedVariable === 42);+ expect(result).to.be.eql(false);+ };+ try {+ await Promise.all([survive(browser), remove(browser)]);} finally {browser.close();}