Skip to content

🕷️ The BEST Web Scraping framework that handles everything from a single request to a full-scale crawl! Don't be shy, join here: https://discord.gg/EMgGbDceNQ and follow here for daily tips and tricks: https://x.com/Scrapling_dev

Adding the JavaScript bypasses files for stealth mode

Karim shoairaadae4a25b37
diff --git a/scrapling/engines/bypasses/navigator_plugins.js b/scrapling/engines/bypasses/navigator_plugins.js
new file mode 100644
index 0000000..653fa5f
--- /dev/null
+++ b/scrapling/engines/bypasses/navigator_plugins.js
@@ -0,0 +1,40 @@
+if(navigator.plugins.length == 0){
+ Object.defineProperty(navigator, 'plugins', {
+ get: () => {
+ const PDFViewerPlugin = Object.create(Plugin.prototype, {
+ description: { value: 'Portable Document Format', enumerable: false },
+ filename: { value: 'internal-pdf-viewer', enumerable: false },
+ name: { value: 'PDF Viewer', enumerable: false },
+ });
+ const ChromePDFViewer = Object.create(Plugin.prototype, {
+ description: { value: 'Portable Document Format', enumerable: false },
+ filename: { value: 'internal-pdf-viewer', enumerable: false },
+ name: { value: 'Chrome PDF Viewer', enumerable: false },
+ });
+ const ChromiumPDFViewer = Object.create(Plugin.prototype, {
+ description: { value: 'Portable Document Format', enumerable: false },
+ filename: { value: 'internal-pdf-viewer', enumerable: false },
+ name: { value: 'Chromium PDF Viewer', enumerable: false },
+ });
+ const EdgePDFViewer = Object.create(Plugin.prototype, {
+ description: { value: 'Portable Document Format', enumerable: false },
+ filename: { value: 'internal-pdf-viewer', enumerable: false },
+ name: { value: 'Microsoft Edge PDF Viewer', enumerable: false },
+ });
+ const WebKitPDFPlugin = Object.create(Plugin.prototype, {
+ description: { value: 'Portable Document Format', enumerable: false },
+ filename: { value: 'internal-pdf-viewer', enumerable: false },
+ name: { value: 'WebKit built-in PDF', enumerable: false },
+ });
+
+ return Object.create(PluginArray.prototype, {
+ length: { value: 5 },
+ 0: { value: PDFViewerPlugin },
+ 1: { value: ChromePDFViewer },
+ 2: { value: ChromiumPDFViewer },
+ 3: { value: EdgePDFViewer },
+ 4: { value: WebKitPDFPlugin },
+ });
+ },
+ });
+}
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/notification_permission.js b/scrapling/engines/bypasses/notification_permission.js
new file mode 100644
index 0000000..0c9c676
--- /dev/null
+++ b/scrapling/engines/bypasses/notification_permission.js
@@ -0,0 +1,5 @@
+// Bypasses `notificationIsDenied` test in creepsjs's 'Like Headless' sections
+const isSecure = document.location.protocol.startsWith('https')
+if (isSecure){
+ Object.defineProperty(Notification, 'permission', {get: () => 'default'})
+}
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/pdf_viewer.js b/scrapling/engines/bypasses/pdf_viewer.js
new file mode 100644
index 0000000..4c88702
--- /dev/null
+++ b/scrapling/engines/bypasses/pdf_viewer.js
@@ -0,0 +1,5 @@
+// PDF viewer enabled
+// Bypasses `pdfIsDisabled` test in creepsjs's 'Like Headless' sections
+Object.defineProperty(navigator, 'pdfViewerEnabled', {
+ get: () => true,
+});
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/playwright_fingerprint.js b/scrapling/engines/bypasses/playwright_fingerprint.js
new file mode 100644
index 0000000..bfba6be
--- /dev/null
+++ b/scrapling/engines/bypasses/playwright_fingerprint.js
@@ -0,0 +1,2 @@
+// Remove playwright fingerprint => https://github.com/microsoft/playwright/commit/c9e673c6dca746384338ab6bb0cf63c7e7caa9b2#diff-087773eea292da9db5a3f27de8f1a2940cdb895383ad750c3cd8e01772a35b40R915
+delete __pwInitScripts;
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/screen_props.js b/scrapling/engines/bypasses/screen_props.js
new file mode 100644
index 0000000..5056b4b
--- /dev/null
+++ b/scrapling/engines/bypasses/screen_props.js
@@ -0,0 +1,27 @@
+const windowScreenProps = {
+ // Dimensions
+ innerHeight: 0,
+ innerWidth: 0,
+ outerHeight: 754,
+ outerWidth: 1313,
+
+ // Position
+ screenX: 19,
+ pageXOffset: 0,
+ pageYOffset: 0,
+
+ // Display
+ devicePixelRatio: 2
+};
+
+try {
+ for (const [prop, value] of Object.entries(windowScreenProps)) {
+ if (value > 0) {
+ // The 0 values are introduced by collecting in the hidden iframe.
+ // They are document sizes anyway so no need to test them or inject them.
+ window[prop] = value;
+ }
+ }
+} catch (e) {
+ console.warn(e);
+};
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/webdriver_fully.js b/scrapling/engines/bypasses/webdriver_fully.js
new file mode 100644
index 0000000..4bda260
--- /dev/null
+++ b/scrapling/engines/bypasses/webdriver_fully.js
@@ -0,0 +1,27 @@
+// Create a function that looks like a native getter
+const nativeGetter = function get webdriver() {
+ return false;
+};
+
+// Copy over native function properties
+Object.defineProperties(nativeGetter, {
+ name: { value: 'get webdriver', configurable: true },
+ length: { value: 0, configurable: true },
+ toString: {
+ value: function() {
+ return `function get webdriver() { [native code] }`;
+ },
+ configurable: true
+ }
+});
+
+// Make it look native
+Object.setPrototypeOf(nativeGetter, Function.prototype);
+
+// Apply the modified descriptor
+Object.defineProperty(Navigator.prototype, 'webdriver', {
+ get: nativeGetter,
+ set: undefined,
+ enumerable: true,
+ configurable: true
+});
\ No newline at end of file
diff --git a/scrapling/engines/bypasses/window_chrome.js b/scrapling/engines/bypasses/window_chrome.js
new file mode 100644
index 0000000..ba63a9c
--- /dev/null
+++ b/scrapling/engines/bypasses/window_chrome.js
@@ -0,0 +1,213 @@
+// To escape `HEADCHR_CHROME_OBJ` test in headless mode => https://github.com/antoinevastel/fp-collect/blob/master/src/fpCollect.js#L322
+// Faking window.chrome fully
+
+if (!window.chrome) {
+ // First, save all existing properties
+ const originalKeys = Object.getOwnPropertyNames(window);
+ const tempObj = {};
+
+ // Recreate all properties in original order
+ for (const key of originalKeys) {
+ const descriptor = Object.getOwnPropertyDescriptor(window, key);
+ const value = window[key];
+ // delete window[key];
+ Object.defineProperty(tempObj, key, descriptor);
+ }
+
+ // Use the exact property descriptor found in headful Chrome
+ // fetch it via `Object.getOwnPropertyDescriptor(window, 'chrome')`
+ const mockChrome = {
+ loadTimes: {},
+ csi: {},
+ app: {
+ isInstalled: false
+ },
+ // Add other Chrome-specific properties
+ };
+
+ Object.defineProperty(tempObj, 'chrome', {
+ writable: true,
+ enumerable: true,
+ configurable: false,
+ value: mockChrome
+ });
+ for (const key of Object.getOwnPropertyNames(tempObj)) {
+ try {
+ Object.defineProperty(window, key,
+ Object.getOwnPropertyDescriptor(tempObj, key));
+ } catch (e) {}
+ };
+ // todo: solve this
+ // Using line below bypasses the hasHighChromeIndex test in creepjs ==> https://github.com/abrahamjuliot/creepjs/blob/master/src/headless/index.ts#L121
+ // Chrome object have to be in the end of the window properties
+ // Object.assign(window, tempObj);
+ // But makes window.chrome unreadable on 'https://bot.sannysoft.com/'
+}
+
+// That means we're running headful and don't need to mock anything
+if ('app' in window.chrome) {
+ return; // Nothing to do here
+}
+const makeError = {
+ ErrorInInvocation: fn => {
+ const err = new TypeError(`Error in invocation of app.${fn}()`);
+ return utils.stripErrorWithAnchor(
+ err,
+ `at ${fn} (eval at <anonymous>`,
+ );
+ },
+};
+// check with: `JSON.stringify(window.chrome['app'])`
+const STATIC_DATA = JSON.parse(
+ `
+{
+ "isInstalled": false,
+ "InstallState": {
+ "DISABLED": "disabled",
+ "INSTALLED": "installed",
+ "NOT_INSTALLED": "not_installed"
+ },
+ "RunningState": {
+ "CANNOT_RUN": "cannot_run",
+ "READY_TO_RUN": "ready_to_run",
+ "RUNNING": "running"
+ }
+}
+ `.trim(),
+ );
+window.chrome.app = {
+ ...STATIC_DATA,
+
+ get isInstalled() {
+ return false;
+ },
+
+ getDetails: function getDetails() {
+ if (arguments.length) {
+ throw makeError.ErrorInInvocation(`getDetails`);
+ }
+ return null;
+ },
+ getIsInstalled: function getDetails() {
+ if (arguments.length) {
+ throw makeError.ErrorInInvocation(`getIsInstalled`);
+ }
+ return false;
+ },
+ runningState: function getDetails() {
+ if (arguments.length) {
+ throw makeError.ErrorInInvocation(`runningState`);
+ }
+ return 'cannot_run';
+ },
+};
+// Check that the Navigation Timing API v1 is available, we need that
+if (!window.performance || !window.performance.timing) {
+ return;
+}
+const {timing} = window.performance;
+window.chrome.csi = function () {
+ return {
+ onloadT: timing.domContentLoadedEventEnd,
+ startE: timing.navigationStart,
+ pageT: Date.now() - timing.navigationStart,
+ tran: 15, // Transition type or something
+ };
+};
+if (!window.PerformancePaintTiming){
+ return;
+}
+const {performance} = window;
+// Some stuff is not available on about:blank as it requires a navigation to occur,
+// let's harden the code to not fail then:
+const ntEntryFallback = {
+ nextHopProtocol: 'h2',
+ type: 'other',
+};
+
+// The API exposes some funky info regarding the connection
+const protocolInfo = {
+ get connectionInfo() {
+ const ntEntry =
+ performance.getEntriesByType('navigation')[0] || ntEntryFallback;
+ return ntEntry.nextHopProtocol;
+ },
+ get npnNegotiatedProtocol() {
+ // NPN is deprecated in favor of ALPN, but this implementation returns the
+ // HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.
+ const ntEntry =
+ performance.getEntriesByType('navigation')[0] || ntEntryFallback;
+ return ['h2', 'hq'].includes(ntEntry.nextHopProtocol)
+ ? ntEntry.nextHopProtocol
+ : 'unknown';
+ },
+ get navigationType() {
+ const ntEntry =
+ performance.getEntriesByType('navigation')[0] || ntEntryFallback;
+ return ntEntry.type;
+ },
+ get wasAlternateProtocolAvailable() {
+ // The Alternate-Protocol header is deprecated in favor of Alt-Svc
+ // (https://www.mnot.net/blog/2016/03/09/alt-svc), so technically this
+ // should always return false.
+ return false;
+ },
+ get wasFetchedViaSpdy() {
+ // SPDY is deprecated in favor of HTTP/2, but this implementation returns
+ // true for HTTP/2 or HTTP2+QUIC/39 as well.
+ const ntEntry =
+ performance.getEntriesByType('navigation')[0] || ntEntryFallback;
+ return ['h2', 'hq'].includes(ntEntry.nextHopProtocol);
+ },
+ get wasNpnNegotiated() {
+ // NPN is deprecated in favor of ALPN, but this implementation returns true
+ // for HTTP/2 or HTTP2+QUIC/39 requests negotiated via ALPN.
+ const ntEntry =
+ performance.getEntriesByType('navigation')[0] || ntEntryFallback;
+ return ['h2', 'hq'].includes(ntEntry.nextHopProtocol);
+ },
+};
+
+// Truncate number to specific number of decimals, most of the `loadTimes` stuff has 3
+function toFixed(num, fixed) {
+ var re = new RegExp('^-?\\d+(?:.\\d{0,' + (fixed || -1) + '})?');
+ return num.toString().match(re)[0];
+}
+
+const timingInfo = {
+ get firstPaintAfterLoadTime() {
+ // This was never actually implemented and always returns 0.
+ return 0;
+ },
+ get requestTime() {
+ return timing.navigationStart / 1000;
+ },
+ get startLoadTime() {
+ return timing.navigationStart / 1000;
+ },
+ get commitLoadTime() {
+ return timing.responseStart / 1000;
+ },
+ get finishDocumentLoadTime() {
+ return timing.domContentLoadedEventEnd / 1000;
+ },
+ get finishLoadTime() {
+ return timing.loadEventEnd / 1000;
+ },
+ get firstPaintTime() {
+ const fpEntry = performance.getEntriesByType('paint')[0] || {
+ startTime: timing.loadEventEnd / 1000, // Fallback if no navigation occured (`about:blank`)
+ };
+ return toFixed(
+ (fpEntry.startTime + performance.timeOrigin) / 1000,
+ 3,
+ );
+ },
+};
+
+window.chrome.loadTimes = function () {
+ return {
+ ...protocolInfo,
+ ...timingInfo,
+ };
+};
\ No newline at end of file