Skip to content

Crawlee—A web scraping and browser automation library for Node.js to build reliable crawlers. In JavaScript and TypeScript. Extract data for AI, LLMs, RAG, or GPTs. Download HTML, PDF, JPG, PNG, and other files from websites. Works with Puppeteer, Playwright, Cheerio, JSDOM, and raw HTTP. Both headful and headless mode. With proxy rotation.

fixed stealth to override the newPage on the context not on browser itself. Fixed KV test same as in #feature/session-pool.

Petr Patek331f6c360932
diff --git a/src/stealth/stealth.js b/src/stealth/stealth.js
index 5b0120a55..bfcce9228 100644
--- a/src/stealth/stealth.js
+++ b/src/stealth/stealth.js
@@ -41,11 +41,15 @@ export default function applyStealthToBrowser(browser, options) {
const modifiedBrowser = browser;
const opts = _.defaults(options, DEFAULT_STEALTH_OPTIONS);
- const prevNewPage = browser.newPage;
- modifiedBrowser.newPage = async (...args) => {
- const page = await prevNewPage.bind(browser)(...args);
+ const defaultContext = browser.defaultBrowserContext();
+ const contextPrototype = Object.getPrototypeOf(defaultContext);
+
+ const prevNewPage = contextPrototype.newPage;
+
+ contextPrototype.newPage = async function (...args) {
+ const page = await prevNewPage.bind(this)(...args);
await applyStealthTricks(page, opts);
- return Promise.resolve(page);
+ return page;
};
diff --git a/test/key_value_store.js b/test/key_value_store.js
index 85d2dfc51..e93c85fba 100644
--- a/test/key_value_store.js
+++ b/test/key_value_store.js
@@ -501,6 +501,7 @@ describe('KeyValueStore', () => {
const defaultStore = await Apify.openKeyValueStore();
// Uses default value.
+ const oldGet = defaultStore.getValue;
defaultStore.getValue = async key => expect(key).to.be.eql(KEY_VALUE_STORE_KEYS.INPUT);
await Apify.getInput();
@@ -511,6 +512,8 @@ describe('KeyValueStore', () => {
delete process.env[ENV_VARS.LOCAL_STORAGE_DIR];
delete process.env[ENV_VARS.INPUT_KEY];
+
+ defaultStore.getValue = oldGet;
});
});
});
diff --git a/test/stealth/stealth.js b/test/stealth/stealth.js
index 8f05bdfdf..c8bda4892 100644
--- a/test/stealth/stealth.js
+++ b/test/stealth/stealth.js
@@ -2,6 +2,7 @@ import { expect } from 'chai';
import scanner from 'fpscanner';
import Apify from '../../build';
+import { LOCAL_STORAGE_DIR, emptyLocalStorageSubdir } from '../_helper';
const fingerPrintPath = require.resolve('fpcollect/dist/fpCollect.min.js');
@@ -13,6 +14,10 @@ const getFingerPrint = async (page) => {
// we can speed up the test to make the requests to the local static html
describe('Stealth - testing headless chrome hiding tricks', () => {
+ before(async () => {
+ process.env.APIFY_LOCAL_STORAGE_DIR = LOCAL_STORAGE_DIR;
+ await emptyLocalStorageSubdir('key_value_stores/default');
+ });
it('it adds plugins, mimeTypes and passes', async () => {
const browser = await Apify.launchPuppeteer({
stealth: true,
@@ -235,4 +240,27 @@ describe('Stealth - testing headless chrome hiding tricks', () => {
return browser.close();
});
+
+ it('should work in crawler', async () => {
+ const requestList = await Apify.openRequestList('test', ['file://test_html.html']);
+ const values = [];
+ const puppeteerCrawler = new Apify.PuppeteerCrawler({
+ requestList,
+ launchPuppeteerOptions: {
+ stealth: true,
+ useChrome: true,
+ headless: true,
+ },
+ handlePageFunction: async ({ page }) => {
+ const fingerprint = await getFingerPrint(page);
+ values.push(fingerprint);
+ },
+ });
+ await puppeteerCrawler.run();
+ const fingerprint = values[0];
+ expect(fingerprint.webDriver).to.be.false; // eslint-disable-line
+ expect(fingerprint.webDriverValue).to.be.undefined; // eslint-disable-line
+
+ console.log(values);
+ });
});