Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): make ad blocking configurable for browser sessions (#4977)

tomsideguide8f288b3c3aed
diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.ts
index 15dee869a..51f762892 100644
--- a/apps/api/src/controllers/v2/browser.ts
+++ b/apps/api/src/controllers/v2/browser.ts
@@ -44,6 +44,7 @@ export const browserCreateRequestSchema = z.object({
recordSession: z.boolean().default(false),
zeroDataRetention: z.boolean().optional(),
integration: integrationSchema.optional().transform(value => value || null),
+ blockAds: z.boolean().default(true),
profile: z
.object({
name: browserProfileNameSchema,
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index 8fbbabc61..aa7e1e3cd 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -52,6 +52,10 @@ import { integrationSchema } from "../../utils/integration";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { getScrapeJobAccess } from "../../lib/operational-job-access";
import { readScrapeJobState } from "../../lib/job-state-store";
+import {
+ type BrowserOptions,
+ browserOptionsFromScrape,
+} from "../../lib/browser-options";
import { scrapeQueue } from "../../services/worker/nuq-router";
const browserExecuteRequestSchema = z
@@ -264,17 +268,19 @@ async function scrapeInteractInternal(
"Replay context is unavailable. Supply url or an existingSessionId to continue without retained scrape context.",
});
}
- const profile =
- state?.profile ??
- (nuqJob?.data.mode === "single_urls"
- ? nuqJob.data.scrapeOptions.profile
- : undefined);
+ const browserOptions =
+ state?.browser ??
+ browserOptionsFromScrape(
+ nuqJob?.data.mode === "single_urls"
+ ? nuqJob.data.scrapeOptions
+ : { profile: state?.profile },
+ );
const created = await createSessionForScrape(
req,
scrapeId,
replayContext,
logger,
- profile,
+ browserOptions,
zeroDataRetention,
);
if (created.error === true) {
@@ -470,14 +476,14 @@ async function createSessionForScrape(
ReturnType<typeof buildReplayContextFromScrape>["context"]
>,
logger: typeof _logger,
- profile: { name: string; saveChanges: boolean } | undefined,
+ browserOptions: BrowserOptions,
zeroDataRetention: boolean,
) {
try {
const { session } = await createBrowserSession(req, {
...browserCreateRequestSchema.parse({}),
+ ...browserOptions,
scrapeId,
- profile,
zeroDataRetention,
initialize: async browserId => {
const replay = await executeHangarBrowser(browserId, {
diff --git a/apps/api/src/lib/browser-lifecycle.ts b/apps/api/src/lib/browser-lifecycle.ts
index 651bd07b6..2b4cd1673 100644
--- a/apps/api/src/lib/browser-lifecycle.ts
+++ b/apps/api/src/lib/browser-lifecycle.ts
@@ -124,6 +124,7 @@ export async function createBrowserSession(
streamWebView: boolean;
recordSession: boolean;
zeroDataRetention?: boolean;
+ blockAds: boolean;
profile?: { name: string; saveChanges: boolean };
scrapeId?: string;
shouldBill?: boolean;
diff --git a/apps/api/src/lib/browser-options.test.ts b/apps/api/src/lib/browser-options.test.ts
new file mode 100644
index 000000000..f5848aa1d
--- /dev/null
+++ b/apps/api/src/lib/browser-options.test.ts
@@ -0,0 +1,12 @@
+import { hangarBrowserOptions } from "./browser-options";
+
+describe("hangarBrowserOptions", () => {
+ it("maps blockAds to Hangar's uBlock extension", () => {
+ expect(hangarBrowserOptions({ blockAds: true })).toEqual({
+ extensions: ["ublock"],
+ });
+ expect(hangarBrowserOptions({ blockAds: false })).toEqual({
+ extensions: [],
+ });
+ });
+});
diff --git a/apps/api/src/lib/browser-options.ts b/apps/api/src/lib/browser-options.ts
new file mode 100644
index 000000000..b0f43d6ab
--- /dev/null
+++ b/apps/api/src/lib/browser-options.ts
@@ -0,0 +1,48 @@
+/**
+ * Browser behavior shared by /v2/browser sessions and sessions created from a
+ * scrape. Option names and defaults follow the scrape API so an interact
+ * session behaves like the page the scrape saw.
+ */
+export type BrowserOptions = {
+ blockAds: boolean;
+ profile?: { name: string; saveChanges: boolean };
+};
+
+export function browserOptionsFromScrape(options: {
+ blockAds?: boolean;
+ profile?: { name: string; saveChanges: boolean };
+}): BrowserOptions {
+ return {
+ blockAds: options.blockAds ?? true,
+ ...(options.profile ? { profile: options.profile } : {}),
+ };
+}
+
+export function isBrowserOptions(value: unknown): value is BrowserOptions {
+ if (!value || typeof value !== "object") return false;
+ const { blockAds, profile } = value as Record<string, unknown>;
+ return (
+ typeof blockAds === "boolean" &&
+ (profile === undefined ||
+ (typeof profile === "object" &&
+ profile !== null &&
+ typeof (profile as any).name === "string" &&
+ typeof (profile as any).saveChanges === "boolean"))
+ );
+}
+
+/** The Hangar create-request fields that carry these options. */
+export function hangarBrowserOptions(options: BrowserOptions) {
+ return {
+ // uBlock Origin Lite covers ads, trackers and cookie notices.
+ extensions: options.blockAds ? ["ublock"] : [],
+ ...(options.profile
+ ? {
+ profile: {
+ name: options.profile.name,
+ save_changes: options.profile.saveChanges,
+ },
+ }
+ : {}),
+ };
+}
diff --git a/apps/api/src/lib/hangar.ts b/apps/api/src/lib/hangar.ts
index 221a7d8d0..d1ce12109 100644
--- a/apps/api/src/lib/hangar.ts
+++ b/apps/api/src/lib/hangar.ts
@@ -1,4 +1,5 @@
import { config } from "../config";
+import { type BrowserOptions, hangarBrowserOptions } from "./browser-options";
export interface HangarBrowser {
id: string;
@@ -106,12 +107,11 @@ export const stopHangarBrowser = (id: string) =>
export async function createHangarBrowser(
key: string,
teamId: string,
- options: {
+ options: BrowserOptions & {
ttl: number;
activityTtl: number;
streamWebView: boolean;
recordSession: boolean;
- profile?: { name: string; saveChanges: boolean };
},
): Promise<HangarCreated> {
const body = {
@@ -124,14 +124,7 @@ export async function createHangarBrowser(
},
recording: { enabled: options.recordSession },
execution: { enabled: true },
- ...(options.profile
- ? {
- profile: {
- name: options.profile.name,
- save_changes: options.profile.saveChanges,
- },
- }
- : {}),
+ ...hangarBrowserOptions(options),
};
let created: HangarCreated | undefined;
for (let attempt = 0; attempt < 3; attempt++) {
diff --git a/apps/api/src/lib/job-state-store.ts b/apps/api/src/lib/job-state-store.ts
index 7d7a90f66..021f4405b 100644
--- a/apps/api/src/lib/job-state-store.ts
+++ b/apps/api/src/lib/job-state-store.ts
@@ -3,6 +3,7 @@ import { getBigtableTable } from "./bigtable-client";
import { saltedUuidV7RowKey } from "./bigtable-row-key";
import { setSpanAttributes, withSpan } from "./otel-tracer";
import type { ScrapeReplayContext } from "./scrape-interact/scrape-replay";
+import { type BrowserOptions, isBrowserOptions } from "./browser-options";
const QUALIFIER = "v";
const FAMILY = "s";
@@ -17,6 +18,8 @@ type ScrapeJobState = {
creditsBilled: number;
error?: string;
replay?: ScrapeReplayContext;
+ browser?: BrowserOptions;
+ /** Written before `browser`; unused once that state has expired. */
profile?: { name: string; saveChanges: boolean };
origin?: string;
};
@@ -75,6 +78,8 @@ function parseScrapeState(value: Buffer | string): ScrapeJobState {
typeof (candidate.replay as any).targetUrl === "string" &&
isFiniteNumber((candidate.replay as any).waitForMs) &&
Array.isArray((candidate.replay as any).actions))) &&
+ (candidate.browser === undefined ||
+ isBrowserOptions(candidate.browser)) &&
(candidate.profile === undefined ||
(typeof candidate.profile === "object" &&
candidate.profile !== null &&
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index e95a29f50..2087f536b 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -39,6 +39,7 @@ import {
writeScrapeJobState,
} from "../../lib/job-state-store";
import { buildReplayContextFromScrape } from "../../lib/scrape-interact/scrape-replay";
+import { browserOptionsFromScrape } from "../../lib/browser-options";
import {
initializeRequestCredits,
recordRequestCredits,
@@ -770,9 +771,7 @@ async function logScrapeInternal(
: {
...(scrape.error ? { error: scrape.error } : {}),
...(replay ? { replay } : {}),
- ...(scrape.options.profile
- ? { profile: scrape.options.profile }
- : {}),
+ browser: browserOptionsFromScrape(scrape.options),
...(typeof (scrape.options as any).origin === "string"
? { origin: (scrape.options as any).origin }
: {}),
diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts
index 5f62d3c61..40c7c6bc2 100644
--- a/apps/js-sdk/firecrawl/src/v2/client.ts
+++ b/apps/js-sdk/firecrawl/src/v2/client.ts
@@ -641,7 +641,7 @@ export class FirecrawlClient {
// Browser
/**
* Create a new browser session.
- * @param args Session options (ttl, activityTtl, streamWebView, profile).
+ * @param args Session options (ttl, activityTtl, streamWebView, blockAds, profile).
* @returns Session id, CDP URL, live view URL, and expiration time.
*/
async browser(
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/browser.ts b/apps/js-sdk/firecrawl/src/v2/methods/browser.ts
index 032f1e99c..fa24a8e90 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/browser.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/browser.ts
@@ -16,6 +16,8 @@ export async function browser(
ttl?: number;
activityTtl?: number;
streamWebView?: boolean;
+ /** Block ads, trackers and cookie notices, as in scrape (default true). */
+ blockAds?: boolean;
profile?: {
name: string;
saveChanges?: boolean;
@@ -28,6 +30,7 @@ export async function browser(
if (args.ttl != null) body.ttl = args.ttl;
if (args.activityTtl != null) body.activityTtl = args.activityTtl;
if (args.streamWebView != null) body.streamWebView = args.streamWebView;
+ if (args.blockAds != null) body.blockAds = args.blockAds;
if (args.profile != null) body.profile = args.profile;
if (args.integration != null) body.integration = args.integration;
if (args.origin) body.origin = args.origin;
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 9f3d5c0c7..0cd74c680 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -1741,6 +1741,7 @@ class FirecrawlClient:
ttl: Optional[int] = None,
activity_ttl: Optional[int] = None,
stream_web_view: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
profile: Optional[Dict[str, Any]] = None,
):
"""Create a new browser session.
@@ -1749,6 +1750,7 @@ class FirecrawlClient:
ttl: Total time-to-live in seconds (30-3600, default 300)
activity_ttl: Inactivity TTL in seconds (10-3600)
stream_web_view: Whether to enable webview streaming
+ block_ads: Block ads, trackers and cookie notices (default ``True``)
profile: Profile config with ``name`` (str) and
optional ``save_changes`` (bool, default ``True``)
@@ -1760,6 +1762,7 @@ class FirecrawlClient:
ttl=ttl,
activity_ttl=activity_ttl,
stream_web_view=stream_web_view,
+ block_ads=block_ads,
profile=profile,
)
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index ca58054ea..86ee3db00 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -949,6 +949,7 @@ class AsyncFirecrawlClient:
ttl: Optional[int] = None,
activity_ttl: Optional[int] = None,
stream_web_view: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
profile: Optional[Dict[str, Any]] = None,
):
"""Create a new browser session.
@@ -957,6 +958,7 @@ class AsyncFirecrawlClient:
ttl: Total time-to-live in seconds (30-3600, default 300)
activity_ttl: Inactivity TTL in seconds (10-3600)
stream_web_view: Whether to enable webview streaming
+ block_ads: Block ads, trackers and cookie notices (default ``True``)
profile: Profile config with ``name`` (str) and
optional ``save_changes`` (bool, default ``True``)
@@ -968,6 +970,7 @@ class AsyncFirecrawlClient:
ttl=ttl,
activity_ttl=activity_ttl,
stream_web_view=stream_web_view,
+ block_ads=block_ads,
profile=profile,
)
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/browser.py b/apps/python-sdk/firecrawl/v2/methods/aio/browser.py
index 27c8b8bcc..6883e264f 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/browser.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/browser.py
@@ -57,6 +57,7 @@ async def browser(
ttl: Optional[int] = None,
activity_ttl: Optional[int] = None,
stream_web_view: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
profile: Optional[Dict[str, Any]] = None,
) -> BrowserCreateResponse:
"""Create a new browser session.
@@ -66,6 +67,7 @@ async def browser(
ttl: Total time-to-live in seconds (30-3600, default 300)
activity_ttl: Inactivity TTL in seconds (10-3600)
stream_web_view: Whether to enable webview streaming
+ block_ads: Block ads, trackers and cookie notices (default ``True``)
profile: Profile config with ``name`` (str) and
optional ``save_changes`` (bool, default ``True``)
@@ -79,6 +81,8 @@ async def browser(
body["activityTtl"] = activity_ttl
if stream_web_view is not None:
body["streamWebView"] = stream_web_view
+ if block_ads is not None:
+ body["blockAds"] = block_ads
if profile is not None:
body["profile"] = {
"name": profile["name"],
diff --git a/apps/python-sdk/firecrawl/v2/methods/browser.py b/apps/python-sdk/firecrawl/v2/methods/browser.py
index 3d75f43a1..78a79dbfe 100644
--- a/apps/python-sdk/firecrawl/v2/methods/browser.py
+++ b/apps/python-sdk/firecrawl/v2/methods/browser.py
@@ -58,6 +58,7 @@ def browser(
ttl: Optional[int] = None,
activity_ttl: Optional[int] = None,
stream_web_view: Optional[bool] = None,
+ block_ads: Optional[bool] = None,
profile: Optional[Dict[str, Any]] = None,
) -> BrowserCreateResponse:
"""Create a new browser session.
@@ -67,6 +68,7 @@ def browser(
ttl: Total time-to-live in seconds (30-3600, default 300)
activity_ttl: Inactivity TTL in seconds (10-3600)
stream_web_view: Whether to enable webview streaming
+ block_ads: Block ads, trackers and cookie notices (default ``True``)
profile: Profile config with ``name`` (str) and
optional ``save_changes`` (bool, default ``True``)
@@ -80,6 +82,8 @@ def browser(
body["activityTtl"] = activity_ttl
if stream_web_view is not None:
body["streamWebView"] = stream_web_view
+ if block_ads is not None:
+ body["blockAds"] = block_ads
if profile is not None:
body["profile"] = {
"name": profile["name"],