Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): point scrapes of search results pages to /search (#5003)

David Tippettb00172b23b4b
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 12be22f40..df97dbafc 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -580,6 +580,18 @@ const configSchema = z.object({
// background and record how its answer compares (lib/branding/jev-shadow.ts).
// Customers always get the LLM's answer.
BRANDING_JEV_SHADOW_PERCENT: z.coerce.number().min(0).max(100).default(0),
+ // Warn scrapes of search engine results pages (google.*/search, ...) that
+ // /search returns those results directly (lib/serp-url.ts).
+ // SERP_SCRAPE_WARNING turns it on for every team, SERP_SCRAPE_WARNING_TEAM_IDS
+ // (comma-separated) for listed teams, and SERP_SCRAPE_WARNING_ROLLOUT_PERCENT
+ // for a stable share of teams.
+ SERP_SCRAPE_WARNING: emptyStringAsUndefined(z.stringbool()),
+ SERP_SCRAPE_WARNING_TEAM_IDS: delimitedList(",").optional(),
+ SERP_SCRAPE_WARNING_ROLLOUT_PERCENT: z.coerce
+ .number()
+ .min(0)
+ .max(100)
+ .default(0),
// AI/ML
MODEL_NAME: z.string().optional(),
diff --git a/apps/api/src/lib/agent-hint-signals.test.ts b/apps/api/src/lib/agent-hint-signals.test.ts
index 6b5a0e598..946444b88 100644
--- a/apps/api/src/lib/agent-hint-signals.test.ts
+++ b/apps/api/src/lib/agent-hint-signals.test.ts
@@ -104,6 +104,39 @@ describe("agent hint signals", () => {
expect(result).not.toHaveProperty("page_path_words");
});
+ it("reports the engine and quoted query for a search results page", () => {
+ const pageOf = (sourceURL: string) =>
+ signalsFor({
+ endpoint: "scrape",
+ response: { success: true, data: { metadata: { sourceURL } } },
+ });
+ expect(pageOf("https://www.bing.com/search?q=web+scraping")).toMatchObject({
+ page_serp_engine: "Bing",
+ page_serp_query: '"web scraping"',
+ });
+ expect(pageOf("https://www.bing.com/maps?q=paris")).not.toHaveProperty(
+ "page_serp_engine",
+ );
+ // A results page that redirected to a consent page still was one.
+ expect(
+ signalsFor({
+ endpoint: "scrape",
+ response: {
+ success: true,
+ data: {
+ metadata: {
+ sourceURL: "https://www.google.com/search?q=firecrawl",
+ url: "https://consent.google.com/m?continue=x",
+ },
+ },
+ },
+ }),
+ ).toMatchObject({
+ page_serp_engine: "Google",
+ page_serp_query: '"firecrawl"',
+ });
+ });
+
it("reports document page counts with the requestable remainder", () => {
expect(
signalsFor({
diff --git a/apps/api/src/lib/agent-hint-signals.ts b/apps/api/src/lib/agent-hint-signals.ts
index 1c99b2771..afd154440 100644
--- a/apps/api/src/lib/agent-hint-signals.ts
+++ b/apps/api/src/lib/agent-hint-signals.ts
@@ -1,3 +1,5 @@
+import { detectSerpPage } from "./serp-url";
+
/**
* Facts about a hint-enabled response, computed locally from the response
* body and request state already in memory. Provider-supplied rules are
@@ -140,6 +142,11 @@ function addPageSignals(
signals.set("page_redirect_to", to);
}
}
+ const serp = detectSerpPage((source ?? final)?.href ?? "");
+ if (serp) {
+ signals.set("page_serp_engine", serp.engine);
+ signals.set("page_serp_query", JSON.stringify(serp.query));
+ }
const page = final ?? source;
if (page) {
signals.set("page_host", page.hostname);
diff --git a/apps/api/src/lib/serp-url.test.ts b/apps/api/src/lib/serp-url.test.ts
new file mode 100644
index 000000000..1fcccfe79
--- /dev/null
+++ b/apps/api/src/lib/serp-url.test.ts
@@ -0,0 +1,109 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const mockConfig = vi.hoisted(() => ({
+ SERP_SCRAPE_WARNING: undefined as boolean | undefined,
+ SERP_SCRAPE_WARNING_TEAM_IDS: undefined as string[] | undefined,
+ SERP_SCRAPE_WARNING_ROLLOUT_PERCENT: 0,
+}));
+
+vi.mock("../config", () => ({ config: mockConfig }));
+
+import {
+ detectSerpPage,
+ serpScrapeWarning,
+ serpScrapeWarningEnabled,
+} from "./serp-url";
+
+describe("detectSerpPage", () => {
+ it.each([
+ ["https://www.google.com/search?q=firecrawl", "Google"],
+ ["https://google.com/search?q=firecrawl&hl=en", "Google"],
+ ["https://www.google.co.uk/search?q=firecrawl", "Google"],
+ ["https://www.google.com/search/?q=firecrawl", "Google"],
+ ["https://www.bing.com/search?q=firecrawl", "Bing"],
+ ["https://duckduckgo.com/?q=firecrawl", "DuckDuckGo"],
+ ["https://html.duckduckgo.com/html/?q=firecrawl", "DuckDuckGo"],
+ ["https://search.brave.com/search?q=firecrawl", "Brave"],
+ ["https://search.yahoo.com/search?p=firecrawl", "Yahoo"],
+ ["https://search.yahoo.co.jp/search?p=firecrawl", "Yahoo"],
+ ["https://search.yahoo.com/search;_ylt=abc?p=firecrawl", "Yahoo"],
+ ["https://lite.duckduckgo.com/lite/?q=firecrawl", "DuckDuckGo"],
+ ["https://www.baidu.com/s?wd=firecrawl", "Baidu"],
+ ["https://yandex.ru/search/?text=firecrawl", "Yandex"],
+ ["https://www.ecosia.org/search?q=firecrawl", "Ecosia"],
+ ["https://www.startpage.com/sp/search?query=firecrawl", "Startpage"],
+ ])("detects %s as %s", (url, engine) => {
+ expect(detectSerpPage(url)).toEqual({ engine, query: "firecrawl" });
+ });
+
+ it("decodes the query and caps its length", () => {
+ expect(
+ detectSerpPage("https://www.google.com/search?q=web+scraping%20api"),
+ ).toEqual({ engine: "Google", query: "web scraping api" });
+ const long = "a".repeat(500);
+ expect(
+ detectSerpPage(`https://www.bing.com/search?q=${long}`)?.query,
+ ).toHaveLength(200);
+ });
+
+ it.each([
+ // Other pages on search engine domains
+ "https://www.google.com/",
+ "https://www.google.com/maps/place/Paris",
+ "https://www.google.com/search",
+ "https://www.google.com/search?q=",
+ "https://scholar.google.com/scholar?q=firecrawl",
+ "https://docs.google.com/document/d/abc/edit",
+ "https://www.bing.com/maps?q=paris",
+ "https://duckduckgo.com/about",
+ "https://www.baidu.com/s",
+ "https://duckduckgo.com/about?q=firecrawl",
+ "https://search.yahoo.com/searchfoo?p=firecrawl",
+ // Look-alike hosts
+ "https://google.example.com/search?q=firecrawl",
+ "https://notgoogle.com/search?q=firecrawl",
+ "https://example.com/search?q=firecrawl",
+ // Not a URL
+ "not a url",
+ "ftp://www.google.com/search?q=firecrawl",
+ ])("ignores %s", url => {
+ expect(detectSerpPage(url)).toBeNull();
+ });
+});
+
+describe("serpScrapeWarning", () => {
+ it("names the engine and points to /search", () => {
+ const warning = serpScrapeWarning("Google");
+ expect(warning).toContain("Google search results page");
+ expect(warning).toContain("/search endpoint");
+ expect(warning).toContain("https://docs.firecrawl.dev/features/search");
+ });
+});
+
+describe("serpScrapeWarningEnabled", () => {
+ beforeEach(() => {
+ mockConfig.SERP_SCRAPE_WARNING = undefined;
+ mockConfig.SERP_SCRAPE_WARNING_TEAM_IDS = undefined;
+ mockConfig.SERP_SCRAPE_WARNING_ROLLOUT_PERCENT = 0;
+ });
+
+ it("is off by default", () => {
+ expect(serpScrapeWarningEnabled("team-a")).toBe(false);
+ });
+
+ it("is on for every team when SERP_SCRAPE_WARNING is set", () => {
+ mockConfig.SERP_SCRAPE_WARNING = true;
+ expect(serpScrapeWarningEnabled("team-a")).toBe(true);
+ });
+
+ it("is on only for listed teams", () => {
+ mockConfig.SERP_SCRAPE_WARNING_TEAM_IDS = ["team-a"];
+ expect(serpScrapeWarningEnabled("team-a")).toBe(true);
+ expect(serpScrapeWarningEnabled("team-b")).toBe(false);
+ });
+
+ it("follows the rollout percent", () => {
+ mockConfig.SERP_SCRAPE_WARNING_ROLLOUT_PERCENT = 100;
+ expect(serpScrapeWarningEnabled("team-a")).toBe(true);
+ });
+});
diff --git a/apps/api/src/lib/serp-url.ts b/apps/api/src/lib/serp-url.ts
new file mode 100644
index 000000000..0c7b26bf9
--- /dev/null
+++ b/apps/api/src/lib/serp-url.ts
@@ -0,0 +1,115 @@
+import { config } from "../config";
+import { sampled } from "./rollout";
+import { parseHostname } from "./url-utils";
+
+type SerpEngine =
+ | "Google"
+ | "Bing"
+ | "DuckDuckGo"
+ | "Brave"
+ | "Yahoo"
+ | "Baidu"
+ | "Yandex"
+ | "Ecosia"
+ | "Startpage";
+
+type SerpPage = { engine: SerpEngine; query: string };
+
+const MAX_QUERY_CHARS = 200;
+
+function queryParam(params: URLSearchParams, ...names: string[]): string {
+ for (const name of names) {
+ const value = (params.get(name) ?? "").trim();
+ if (value) return value.slice(0, MAX_QUERY_CHARS);
+ }
+ return "";
+}
+
+function engineAndQuery(url: URL): SerpPage | null {
+ const host = url.hostname.toLowerCase();
+ const parsedHost = parseHostname(host);
+ const domain = parsedHost.domain ?? "";
+ const domainWithoutSuffix = parsedHost.domainWithoutSuffix ?? "";
+ const subdomain = parsedHost.subdomain ?? "";
+ const path = url.pathname.replace(/\/+$/, "") || "/";
+ const params = url.searchParams;
+ const bareOrWww = subdomain === "" || subdomain === "www";
+ const page = (engine: SerpEngine, matches: boolean, ...names: string[]) => {
+ const query = matches ? queryParam(params, ...names) : "";
+ return query ? { engine, query } : null;
+ };
+
+ if (domainWithoutSuffix === "google" && bareOrWww) {
+ return page("Google", path === "/search", "q");
+ }
+ if (domain === "bing.com") {
+ return page("Bing", path === "/search", "q");
+ }
+ if (domain === "duckduckgo.com") {
+ return page(
+ "DuckDuckGo",
+ path === "/" || path === "/html" || path === "/lite",
+ "q",
+ );
+ }
+ if (host === "search.brave.com") {
+ return page("Brave", path === "/search", "q");
+ }
+ if (
+ domainWithoutSuffix === "yahoo" &&
+ (subdomain === "search" || subdomain.endsWith(".search"))
+ ) {
+ // Yahoo appends tracking after a semicolon: /search;_ylt=...
+ return page("Yahoo", /^\/search(?:[/;]|$)/.test(path), "p", "q");
+ }
+ if (domain === "baidu.com" && bareOrWww) {
+ return page("Baidu", path === "/s", "wd", "word");
+ }
+ if (domainWithoutSuffix === "yandex" && bareOrWww) {
+ return page("Yandex", path === "/search", "text");
+ }
+ if (domain === "ecosia.org") {
+ return page("Ecosia", path === "/search", "q");
+ }
+ if (domain === "startpage.com") {
+ return page(
+ "Startpage",
+ path === "/do/search" || path === "/sp/search",
+ "q",
+ "query",
+ );
+ }
+ return null;
+}
+
+/**
+ * Returns the search engine and query when the URL is a results page for a
+ * query (google.*\/search?q=, bing.com/search?q=, ...), otherwise null. Other
+ * pages on those domains (Maps, Shopping product pages, the home page) are not
+ * results pages and return null.
+ */
+export function detectSerpPage(url: string): SerpPage | null {
+ let parsed: URL;
+ try {
+ parsed = new URL(url);
+ } catch {
+ return null;
+ }
+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;
+ return engineAndQuery(parsed);
+}
+
+export function serpScrapeWarning(engine: SerpEngine): string {
+ return `This URL is a ${engine} search results page. Firecrawl's /search endpoint returns ranked results for a query without the page's ads and layout, and can scrape each result in the same call: https://docs.firecrawl.dev/features/search`;
+}
+
+/**
+ * Whether a team sees the SERP scrape warning. SERP_SCRAPE_WARNING turns it on
+ * for every team, SERP_SCRAPE_WARNING_TEAM_IDS for listed teams, and
+ * SERP_SCRAPE_WARNING_ROLLOUT_PERCENT for a stable share of teams.
+ */
+export function serpScrapeWarningEnabled(teamId: string): boolean {
+ if (config.SERP_SCRAPE_WARNING) return true;
+ if (config.SERP_SCRAPE_WARNING_TEAM_IDS?.includes(teamId)) return true;
+ return sampled(`team:${teamId}`, config.SERP_SCRAPE_WARNING_ROLLOUT_PERCENT);
+}
diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.ts
index bfb7d562c..52c41891b 100644
--- a/apps/api/src/services/worker/scrape-worker.ts
+++ b/apps/api/src/services/worker/scrape-worker.ts
@@ -47,6 +47,11 @@ import {
addScrapeJobs,
} from "../queue-jobs";
import { parseHostname } from "../../lib/url-utils";
+import {
+ detectSerpPage,
+ serpScrapeWarning,
+ serpScrapeWarningEnabled,
+} from "../../lib/serp-url";
import { getJobPriority } from "../../lib/job-priority";
import { Document, scrapeOptions, TeamFlags } from "../../controllers/v2/types";
import { hasFormatOfType } from "../../lib/format-utils";
@@ -539,6 +544,26 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {
(doc.warning ? " " + doc.warning : "");
}
+ const billingEndpoint = job.data.billing?.endpoint;
+ if (billingEndpoint === "scrape" || billingEndpoint === "batch_scrape") {
+ const serpPage = detectSerpPage(job.data.url);
+ if (serpPage !== null) {
+ // Logged whether or not the warning is shown, so the rollout can be
+ // compared with teams outside it.
+ const warned = serpScrapeWarningEnabled(job.data.team_id);
+ logger.info("SERP scrape", {
+ serpEngine: serpPage.engine,
+ serpWarningShown: warned,
+ billingEndpoint,
+ });
+ if (warned) {
+ doc.warning =
+ serpScrapeWarning(serpPage.engine) +
+ (doc.warning ? " " + doc.warning : "");
+ }
+ }
+ }
+
const data = {
success: true,
result: {