Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(scrapeURL): only allow disabling the adblock on playwright (FIR-2200) (#1616)
Gergő Móricz4167ec53eb11
diff --git a/apps/api/src/__tests__/snips/scrape.test.ts b/apps/api/src/__tests__/snips/scrape.test.tsindex a27057f78..156e8e15e 100644--- a/apps/api/src/__tests__/snips/scrape.test.ts+++ b/apps/api/src/__tests__/snips/scrape.test.ts@@ -77,24 +77,24 @@ describe("Scrape tests", () => {expect(JSON.stringify(status)).toBe(JSON.stringify(response));}, 60000);- // describe("Ad blocking (f-e dependant)", () => {- // it.concurrent("blocks ads by default", async () => {- // const response = await scrape({- // url: "https://www.allrecipes.com/recipe/18185/yum/",- // });-- // expect(response.markdown).not.toContain(".g.doubleclick.net/");- // }, 30000);-- // it.concurrent("doesn't block ads if explicitly disabled", async () => {- // const response = await scrape({- // url: "https://www.allrecipes.com/recipe/18185/yum/",- // blockAds: false,- // });-- // expect(response.markdown).toMatch(/(\.g\.doubleclick\.net|amazon-adsystem\.com)\//);- // }, 30000);- // });+ describe("Ad blocking (f-e dependant)", () => {+ it.concurrent("blocks ads by default", async () => {+ const response = await scrape({+ url: "https://www.allrecipes.com/recipe/18185/yum/",+ });++ expect(response.markdown).not.toContain(".g.doubleclick.net/");+ }, 30000);++ it.concurrent("doesn't block ads if explicitly disabled", async () => {+ const response = await scrape({+ url: "https://www.allrecipes.com/recipe/18185/yum/",+ blockAds: false,+ });++ expect(response.markdown).toMatch(/(\.g\.doubleclick\.net|amazon-adsystem\.com)\//);+ }, 30000);+ });describe("Change Tracking format", () => {it.concurrent("works", async () => {diff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.tsindex 8bbacb4c0..3e8490c40 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/index.ts@@ -224,7 +224,6 @@ export async function scrapeURLWithFireEngineChromeCDP(mobile: meta.options.mobile,timeout, // TODO: better timeout logicdisableSmartWaitCache: meta.internalOptions.disableSmartWaitCache,- blockAds: meta.options.blockAds,mobileProxy: meta.featureFlags.has("stealthProxy"),saveScrapeResultToGCS: meta.internalOptions.saveScrapeResultToGCS,// TODO: scrollXPathsdiff --git a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.tsindex e0c44e18c..5261b697a 100644--- a/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts+++ b/apps/api/src/scraper/scrapeURL/engines/fire-engine/scrape.ts@@ -12,7 +12,6 @@ export type FireEngineScrapeRequestCommon = {headers?: { [K: string]: string };blockMedia?: boolean; // default: true- blockAds?: boolean; // default: true// pageOptions?: any; // unused, .scrollXPaths is considered on FE side// useProxy?: boolean; // unused, default: true@@ -39,7 +38,6 @@ export type FireEngineScrapeRequestChromeCDP = {blockMedia?: true; // cannot be falsemobile?: boolean;disableSmartWaitCache?: boolean;- blockAds?: boolean; // default: truesaveScrapeResultToGCS?: boolean;};@@ -58,7 +56,6 @@ export type FireEngineScrapeRequestTLSClient = {engine: "tlsclient";atsv?: boolean; // v0 only, default: falsedisableJsDom?: boolean; // v0 only, default: false- // blockAds?: boolean; // default: true};const schema = z.object({diff --git a/apps/api/src/scraper/scrapeURL/engines/index.ts b/apps/api/src/scraper/scrapeURL/engines/index.tsindex fe0ae8c70..c690ac7b0 100644--- a/apps/api/src/scraper/scrapeURL/engines/index.ts+++ b/apps/api/src/scraper/scrapeURL/engines/index.ts@@ -69,6 +69,7 @@ export const featureFlags = ["skipTlsVerification","useFastMode","stealthProxy",+ "disableAdblock",] as const;export type FeatureFlag = (typeof featureFlags)[number];@@ -90,6 +91,7 @@ export const featureFlagOptions: {mobile: { priority: 10 },skipTlsVerification: { priority: 10 },stealthProxy: { priority: 20 },+ disableAdblock: { priority: 10 },} as const;export type EngineScrapeResult = {@@ -160,6 +162,7 @@ export const engineOptions: {skipTlsVerification: false,useFastMode: false,stealthProxy: false,+ disableAdblock: false,},quality: 1000, // cache should always be tried first},@@ -177,6 +180,7 @@ export const engineOptions: {skipTlsVerification: true,useFastMode: false,stealthProxy: false,+ disableAdblock: false,},quality: 50,},@@ -194,6 +198,7 @@ export const engineOptions: {skipTlsVerification: true,useFastMode: false,stealthProxy: false,+ disableAdblock: false,},quality: 45,},@@ -211,6 +216,7 @@ export const engineOptions: {skipTlsVerification: true,useFastMode: false,stealthProxy: true,+ disableAdblock: false,},quality: -1,},@@ -228,6 +234,7 @@ export const engineOptions: {skipTlsVerification: true,useFastMode: false,stealthProxy: true,+ disableAdblock: false,},quality: -5,},@@ -245,6 +252,7 @@ export const engineOptions: {skipTlsVerification: false,useFastMode: false,stealthProxy: false,+ disableAdblock: true,},quality: 40,},@@ -262,6 +270,7 @@ export const engineOptions: {skipTlsVerification: false,useFastMode: false,stealthProxy: true,+ disableAdblock: true,},quality: -10,},@@ -279,6 +288,7 @@ export const engineOptions: {skipTlsVerification: false,useFastMode: false,stealthProxy: false,+ disableAdblock: false,},quality: 20,},@@ -296,6 +306,7 @@ export const engineOptions: {skipTlsVerification: false,useFastMode: true,stealthProxy: false,+ disableAdblock: false,},quality: 10,},@@ -313,6 +324,7 @@ export const engineOptions: {skipTlsVerification: false,useFastMode: true,stealthProxy: true,+ disableAdblock: false,},quality: -15,},@@ -330,6 +342,7 @@ export const engineOptions: {skipTlsVerification: false,useFastMode: true,stealthProxy: false,+ disableAdblock: false,},quality: 5,},@@ -347,6 +360,7 @@ export const engineOptions: {skipTlsVerification: false,useFastMode: true,stealthProxy: true, // kinda...+ disableAdblock: true,},quality: -20,},@@ -364,6 +378,7 @@ export const engineOptions: {skipTlsVerification: false,useFastMode: true,stealthProxy: true, // kinda...+ disableAdblock: true,},quality: -20,},diff --git a/apps/api/src/scraper/scrapeURL/index.ts b/apps/api/src/scraper/scrapeURL/index.tsindex 90873ca6e..2f926a966 100644--- a/apps/api/src/scraper/scrapeURL/index.ts+++ b/apps/api/src/scraper/scrapeURL/index.ts@@ -118,6 +118,10 @@ function buildFeatureFlags(flags.add("docx");}+ if (options.blockAds === false) {+ flags.add("disableAdblock");+ }+return flags;}