Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): make ad blocking configurable for browser sessions (#4977)
tomsideguide8f288b3c3aed
diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex 15dee869a..51f762892 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -44,6 +44,7 @@ export const browserCreateRequestSchema = z.object({recordSession: z.boolean().default(false),zeroDataRetention: z.boolean().optional(),integration: integrationSchema.optional().transform(value => value || null),+ blockAds: z.boolean().default(true),profile: z.object({name: browserProfileNameSchema,diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 8fbbabc61..aa7e1e3cd 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -52,6 +52,10 @@ import { integrationSchema } from "../../utils/integration";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { getScrapeJobAccess } from "../../lib/operational-job-access";import { readScrapeJobState } from "../../lib/job-state-store";+import {+ type BrowserOptions,+ browserOptionsFromScrape,+} from "../../lib/browser-options";import { scrapeQueue } from "../../services/worker/nuq-router";const browserExecuteRequestSchema = z@@ -264,17 +268,19 @@ async function scrapeInteractInternal("Replay context is unavailable. Supply url or an existingSessionId to continue without retained scrape context.",});}- const profile =- state?.profile ??- (nuqJob?.data.mode === "single_urls"- ? nuqJob.data.scrapeOptions.profile- : undefined);+ const browserOptions =+ state?.browser ??+ browserOptionsFromScrape(+ nuqJob?.data.mode === "single_urls"+ ? nuqJob.data.scrapeOptions+ : { profile: state?.profile },+ );const created = await createSessionForScrape(req,scrapeId,replayContext,logger,- profile,+ browserOptions,zeroDataRetention,);if (created.error === true) {@@ -470,14 +476,14 @@ async function createSessionForScrape(ReturnType<typeof buildReplayContextFromScrape>["context"]>,logger: typeof _logger,- profile: { name: string; saveChanges: boolean } | undefined,+ browserOptions: BrowserOptions,zeroDataRetention: boolean,) {try {const { session } = await createBrowserSession(req, {...browserCreateRequestSchema.parse({}),+ ...browserOptions,scrapeId,- profile,zeroDataRetention,initialize: async browserId => {const replay = await executeHangarBrowser(browserId, {diff --git a/apps/api/src/lib/browser-lifecycle.ts b/apps/api/src/lib/browser-lifecycle.tsindex 651bd07b6..2b4cd1673 100644--- a/apps/api/src/lib/browser-lifecycle.ts+++ b/apps/api/src/lib/browser-lifecycle.ts@@ -124,6 +124,7 @@ export async function createBrowserSession(streamWebView: boolean;recordSession: boolean;zeroDataRetention?: boolean;+ blockAds: boolean;profile?: { name: string; saveChanges: boolean };scrapeId?: string;shouldBill?: boolean;diff --git a/apps/api/src/lib/browser-options.test.ts b/apps/api/src/lib/browser-options.test.tsnew file mode 100644index 000000000..f5848aa1d--- /dev/null+++ b/apps/api/src/lib/browser-options.test.ts@@ -0,0 +1,12 @@+import { hangarBrowserOptions } from "./browser-options";++describe("hangarBrowserOptions", () => {+ it("maps blockAds to Hangar's uBlock extension", () => {+ expect(hangarBrowserOptions({ blockAds: true })).toEqual({+ extensions: ["ublock"],+ });+ expect(hangarBrowserOptions({ blockAds: false })).toEqual({+ extensions: [],+ });+ });+});diff --git a/apps/api/src/lib/browser-options.ts b/apps/api/src/lib/browser-options.tsnew file mode 100644index 000000000..b0f43d6ab--- /dev/null+++ b/apps/api/src/lib/browser-options.ts@@ -0,0 +1,48 @@+/**+ * Browser behavior shared by /v2/browser sessions and sessions created from a+ * scrape. Option names and defaults follow the scrape API so an interact+ * session behaves like the page the scrape saw.+ */+export type BrowserOptions = {+ blockAds: boolean;+ profile?: { name: string; saveChanges: boolean };+};++export function browserOptionsFromScrape(options: {+ blockAds?: boolean;+ profile?: { name: string; saveChanges: boolean };+}): BrowserOptions {+ return {+ blockAds: options.blockAds ?? true,+ ...(options.profile ? { profile: options.profile } : {}),+ };+}++export function isBrowserOptions(value: unknown): value is BrowserOptions {+ if (!value || typeof value !== "object") return false;+ const { blockAds, profile } = value as Record<string, unknown>;+ return (+ typeof blockAds === "boolean" &&+ (profile === undefined ||+ (typeof profile === "object" &&+ profile !== null &&+ typeof (profile as any).name === "string" &&+ typeof (profile as any).saveChanges === "boolean"))+ );+}++/** The Hangar create-request fields that carry these options. */+export function hangarBrowserOptions(options: BrowserOptions) {+ return {+ // uBlock Origin Lite covers ads, trackers and cookie notices.+ extensions: options.blockAds ? ["ublock"] : [],+ ...(options.profile+ ? {+ profile: {+ name: options.profile.name,+ save_changes: options.profile.saveChanges,+ },+ }+ : {}),+ };+}diff --git a/apps/api/src/lib/hangar.ts b/apps/api/src/lib/hangar.tsindex 221a7d8d0..d1ce12109 100644--- a/apps/api/src/lib/hangar.ts+++ b/apps/api/src/lib/hangar.ts@@ -1,4 +1,5 @@import { config } from "../config";+import { type BrowserOptions, hangarBrowserOptions } from "./browser-options";export interface HangarBrowser {id: string;@@ -106,12 +107,11 @@ export const stopHangarBrowser = (id: string) =>export async function createHangarBrowser(key: string,teamId: string,- options: {+ options: BrowserOptions & {ttl: number;activityTtl: number;streamWebView: boolean;recordSession: boolean;- profile?: { name: string; saveChanges: boolean };},): Promise<HangarCreated> {const body = {@@ -124,14 +124,7 @@ export async function createHangarBrowser(},recording: { enabled: options.recordSession },execution: { enabled: true },- ...(options.profile- ? {- profile: {- name: options.profile.name,- save_changes: options.profile.saveChanges,- },- }- : {}),+ ...hangarBrowserOptions(options),};let created: HangarCreated | undefined;for (let attempt = 0; attempt < 3; attempt++) {diff --git a/apps/api/src/lib/job-state-store.ts b/apps/api/src/lib/job-state-store.tsindex 7d7a90f66..021f4405b 100644--- a/apps/api/src/lib/job-state-store.ts+++ b/apps/api/src/lib/job-state-store.ts@@ -3,6 +3,7 @@ import { getBigtableTable } from "./bigtable-client";import { saltedUuidV7RowKey } from "./bigtable-row-key";import { setSpanAttributes, withSpan } from "./otel-tracer";import type { ScrapeReplayContext } from "./scrape-interact/scrape-replay";+import { type BrowserOptions, isBrowserOptions } from "./browser-options";const QUALIFIER = "v";const FAMILY = "s";@@ -17,6 +18,8 @@ type ScrapeJobState = {creditsBilled: number;error?: string;replay?: ScrapeReplayContext;+ browser?: BrowserOptions;+ /** Written before `browser`; unused once that state has expired. */profile?: { name: string; saveChanges: boolean };origin?: string;};@@ -75,6 +78,8 @@ function parseScrapeState(value: Buffer | string): ScrapeJobState {typeof (candidate.replay as any).targetUrl === "string" &&isFiniteNumber((candidate.replay as any).waitForMs) &&Array.isArray((candidate.replay as any).actions))) &&+ (candidate.browser === undefined ||+ isBrowserOptions(candidate.browser)) &&(candidate.profile === undefined ||(typeof candidate.profile === "object" &&candidate.profile !== null &&diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex e95a29f50..2087f536b 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -39,6 +39,7 @@ import {writeScrapeJobState,} from "../../lib/job-state-store";import { buildReplayContextFromScrape } from "../../lib/scrape-interact/scrape-replay";+import { browserOptionsFromScrape } from "../../lib/browser-options";import {initializeRequestCredits,recordRequestCredits,@@ -770,9 +771,7 @@ async function logScrapeInternal(: {...(scrape.error ? { error: scrape.error } : {}),...(replay ? { replay } : {}),- ...(scrape.options.profile- ? { profile: scrape.options.profile }- : {}),+ browser: browserOptionsFromScrape(scrape.options),...(typeof (scrape.options as any).origin === "string"? { origin: (scrape.options as any).origin }: {}),diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex 5f62d3c61..40c7c6bc2 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -641,7 +641,7 @@ export class FirecrawlClient {// Browser/*** Create a new browser session.- * @param args Session options (ttl, activityTtl, streamWebView, profile).+ * @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).* @returns Session id, CDP URL, live view URL, and expiration time.*/async browser(diff --git a/apps/js-sdk/firecrawl/src/v2/methods/browser.ts b/apps/js-sdk/firecrawl/src/v2/methods/browser.tsindex 032f1e99c..fa24a8e90 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/browser.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/browser.ts@@ -16,6 +16,8 @@ export async function browser(ttl?: number;activityTtl?: number;streamWebView?: boolean;+ /** Block ads, trackers and cookie notices, as in scrape (default true). */+ blockAds?: boolean;profile?: {name: string;saveChanges?: boolean;@@ -28,6 +30,7 @@ export async function browser(if (args.ttl != null) body.ttl = args.ttl;if (args.activityTtl != null) body.activityTtl = args.activityTtl;if (args.streamWebView != null) body.streamWebView = args.streamWebView;+ if (args.blockAds != null) body.blockAds = args.blockAds;if (args.profile != null) body.profile = args.profile;if (args.integration != null) body.integration = args.integration;if (args.origin) body.origin = args.origin;diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 9f3d5c0c7..0cd74c680 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -1741,6 +1741,7 @@ class FirecrawlClient:ttl: Optional[int] = None,activity_ttl: Optional[int] = None,stream_web_view: Optional[bool] = None,+ block_ads: Optional[bool] = None,profile: Optional[Dict[str, Any]] = None,):"""Create a new browser session.@@ -1749,6 +1750,7 @@ class FirecrawlClient:ttl: Total time-to-live in seconds (30-3600, default 300)activity_ttl: Inactivity TTL in seconds (10-3600)stream_web_view: Whether to enable webview streaming+ block_ads: Block ads, trackers and cookie notices (default ``True``)profile: Profile config with ``name`` (str) andoptional ``save_changes`` (bool, default ``True``)@@ -1760,6 +1762,7 @@ class FirecrawlClient:ttl=ttl,activity_ttl=activity_ttl,stream_web_view=stream_web_view,+ block_ads=block_ads,profile=profile,)diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex ca58054ea..86ee3db00 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -949,6 +949,7 @@ class AsyncFirecrawlClient:ttl: Optional[int] = None,activity_ttl: Optional[int] = None,stream_web_view: Optional[bool] = None,+ block_ads: Optional[bool] = None,profile: Optional[Dict[str, Any]] = None,):"""Create a new browser session.@@ -957,6 +958,7 @@ class AsyncFirecrawlClient:ttl: Total time-to-live in seconds (30-3600, default 300)activity_ttl: Inactivity TTL in seconds (10-3600)stream_web_view: Whether to enable webview streaming+ block_ads: Block ads, trackers and cookie notices (default ``True``)profile: Profile config with ``name`` (str) andoptional ``save_changes`` (bool, default ``True``)@@ -968,6 +970,7 @@ class AsyncFirecrawlClient:ttl=ttl,activity_ttl=activity_ttl,stream_web_view=stream_web_view,+ block_ads=block_ads,profile=profile,)diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/browser.py b/apps/python-sdk/firecrawl/v2/methods/aio/browser.pyindex 27c8b8bcc..6883e264f 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/browser.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/browser.py@@ -57,6 +57,7 @@ async def browser(ttl: Optional[int] = None,activity_ttl: Optional[int] = None,stream_web_view: Optional[bool] = None,+ block_ads: Optional[bool] = None,profile: Optional[Dict[str, Any]] = None,) -> BrowserCreateResponse:"""Create a new browser session.@@ -66,6 +67,7 @@ async def browser(ttl: Total time-to-live in seconds (30-3600, default 300)activity_ttl: Inactivity TTL in seconds (10-3600)stream_web_view: Whether to enable webview streaming+ block_ads: Block ads, trackers and cookie notices (default ``True``)profile: Profile config with ``name`` (str) andoptional ``save_changes`` (bool, default ``True``)@@ -79,6 +81,8 @@ async def browser(body["activityTtl"] = activity_ttlif stream_web_view is not None:body["streamWebView"] = stream_web_view+ if block_ads is not None:+ body["blockAds"] = block_adsif profile is not None:body["profile"] = {"name": profile["name"],diff --git a/apps/python-sdk/firecrawl/v2/methods/browser.py b/apps/python-sdk/firecrawl/v2/methods/browser.pyindex 3d75f43a1..78a79dbfe 100644--- a/apps/python-sdk/firecrawl/v2/methods/browser.py+++ b/apps/python-sdk/firecrawl/v2/methods/browser.py@@ -58,6 +58,7 @@ def browser(ttl: Optional[int] = None,activity_ttl: Optional[int] = None,stream_web_view: Optional[bool] = None,+ block_ads: Optional[bool] = None,profile: Optional[Dict[str, Any]] = None,) -> BrowserCreateResponse:"""Create a new browser session.@@ -67,6 +68,7 @@ def browser(ttl: Total time-to-live in seconds (30-3600, default 300)activity_ttl: Inactivity TTL in seconds (10-3600)stream_web_view: Whether to enable webview streaming+ block_ads: Block ads, trackers and cookie notices (default ``True``)profile: Profile config with ``name`` (str) andoptional ``save_changes`` (bool, default ``True``)@@ -80,6 +82,8 @@ def browser(body["activityTtl"] = activity_ttlif stream_web_view is not None:body["streamWebView"] = stream_web_view+ if block_ads is not None:+ body["blockAds"] = block_adsif profile is not None:body["profile"] = {"name": profile["name"],