Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): point scrapes of search results pages to /search (#5003)
David Tippettb00172b23b4b
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 12be22f40..df97dbafc 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -580,6 +580,18 @@ const configSchema = z.object({// background and record how its answer compares (lib/branding/jev-shadow.ts).// Customers always get the LLM's answer.BRANDING_JEV_SHADOW_PERCENT: z.coerce.number().min(0).max(100).default(0),+ // Warn scrapes of search engine results pages (google.*/search, ...) that+ // /search returns those results directly (lib/serp-url.ts).+ // SERP_SCRAPE_WARNING turns it on for every team, SERP_SCRAPE_WARNING_TEAM_IDS+ // (comma-separated) for listed teams, and SERP_SCRAPE_WARNING_ROLLOUT_PERCENT+ // for a stable share of teams.+ SERP_SCRAPE_WARNING: emptyStringAsUndefined(z.stringbool()),+ SERP_SCRAPE_WARNING_TEAM_IDS: delimitedList(",").optional(),+ SERP_SCRAPE_WARNING_ROLLOUT_PERCENT: z.coerce+ .number()+ .min(0)+ .max(100)+ .default(0),// AI/MLMODEL_NAME: z.string().optional(),diff --git a/apps/api/src/lib/agent-hint-signals.test.ts b/apps/api/src/lib/agent-hint-signals.test.tsindex 6b5a0e598..946444b88 100644--- a/apps/api/src/lib/agent-hint-signals.test.ts+++ b/apps/api/src/lib/agent-hint-signals.test.ts@@ -104,6 +104,39 @@ describe("agent hint signals", () => {expect(result).not.toHaveProperty("page_path_words");});+ it("reports the engine and quoted query for a search results page", () => {+ const pageOf = (sourceURL: string) =>+ signalsFor({+ endpoint: "scrape",+ response: { success: true, data: { metadata: { sourceURL } } },+ });+ expect(pageOf("https://www.bing.com/search?q=web+scraping")).toMatchObject({+ page_serp_engine: "Bing",+ page_serp_query: '"web scraping"',+ });+ expect(pageOf("https://www.bing.com/maps?q=paris")).not.toHaveProperty(+ "page_serp_engine",+ );+ // A results page that redirected to a consent page still was one.+ expect(+ signalsFor({+ endpoint: "scrape",+ response: {+ success: true,+ data: {+ metadata: {+ sourceURL: "https://www.google.com/search?q=firecrawl",+ url: "https://consent.google.com/m?continue=x",+ },+ },+ },+ }),+ ).toMatchObject({+ page_serp_engine: "Google",+ page_serp_query: '"firecrawl"',+ });+ });+it("reports document page counts with the requestable remainder", () => {expect(signalsFor({diff --git a/apps/api/src/lib/agent-hint-signals.ts b/apps/api/src/lib/agent-hint-signals.tsindex 1c99b2771..afd154440 100644--- a/apps/api/src/lib/agent-hint-signals.ts+++ b/apps/api/src/lib/agent-hint-signals.ts@@ -1,3 +1,5 @@+import { detectSerpPage } from "./serp-url";+/*** Facts about a hint-enabled response, computed locally from the response* body and request state already in memory. Provider-supplied rules are@@ -140,6 +142,11 @@ function addPageSignals(signals.set("page_redirect_to", to);}}+ const serp = detectSerpPage((source ?? final)?.href ?? "");+ if (serp) {+ signals.set("page_serp_engine", serp.engine);+ signals.set("page_serp_query", JSON.stringify(serp.query));+ }const page = final ?? source;if (page) {signals.set("page_host", page.hostname);diff --git a/apps/api/src/lib/serp-url.test.ts b/apps/api/src/lib/serp-url.test.tsnew file mode 100644index 000000000..1fcccfe79--- /dev/null+++ b/apps/api/src/lib/serp-url.test.ts@@ -0,0 +1,109 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const mockConfig = vi.hoisted(() => ({+ SERP_SCRAPE_WARNING: undefined as boolean | undefined,+ SERP_SCRAPE_WARNING_TEAM_IDS: undefined as string[] | undefined,+ SERP_SCRAPE_WARNING_ROLLOUT_PERCENT: 0,+}));++vi.mock("../config", () => ({ config: mockConfig }));++import {+ detectSerpPage,+ serpScrapeWarning,+ serpScrapeWarningEnabled,+} from "./serp-url";++describe("detectSerpPage", () => {+ it.each([+ ["https://www.google.com/search?q=firecrawl", "Google"],+ ["https://google.com/search?q=firecrawl&hl=en", "Google"],+ ["https://www.google.co.uk/search?q=firecrawl", "Google"],+ ["https://www.google.com/search/?q=firecrawl", "Google"],+ ["https://www.bing.com/search?q=firecrawl", "Bing"],+ ["https://duckduckgo.com/?q=firecrawl", "DuckDuckGo"],+ ["https://html.duckduckgo.com/html/?q=firecrawl", "DuckDuckGo"],+ ["https://search.brave.com/search?q=firecrawl", "Brave"],+ ["https://search.yahoo.com/search?p=firecrawl", "Yahoo"],+ ["https://search.yahoo.co.jp/search?p=firecrawl", "Yahoo"],+ ["https://search.yahoo.com/search;_ylt=abc?p=firecrawl", "Yahoo"],+ ["https://lite.duckduckgo.com/lite/?q=firecrawl", "DuckDuckGo"],+ ["https://www.baidu.com/s?wd=firecrawl", "Baidu"],+ ["https://yandex.ru/search/?text=firecrawl", "Yandex"],+ ["https://www.ecosia.org/search?q=firecrawl", "Ecosia"],+ ["https://www.startpage.com/sp/search?query=firecrawl", "Startpage"],+ ])("detects %s as %s", (url, engine) => {+ expect(detectSerpPage(url)).toEqual({ engine, query: "firecrawl" });+ });++ it("decodes the query and caps its length", () => {+ expect(+ detectSerpPage("https://www.google.com/search?q=web+scraping%20api"),+ ).toEqual({ engine: "Google", query: "web scraping api" });+ const long = "a".repeat(500);+ expect(+ detectSerpPage(`https://www.bing.com/search?q=${long}`)?.query,+ ).toHaveLength(200);+ });++ it.each([+ // Other pages on search engine domains+ "https://www.google.com/",+ "https://www.google.com/maps/place/Paris",+ "https://www.google.com/search",+ "https://www.google.com/search?q=",+ "https://scholar.google.com/scholar?q=firecrawl",+ "https://docs.google.com/document/d/abc/edit",+ "https://www.bing.com/maps?q=paris",+ "https://duckduckgo.com/about",+ "https://www.baidu.com/s",+ "https://duckduckgo.com/about?q=firecrawl",+ "https://search.yahoo.com/searchfoo?p=firecrawl",+ // Look-alike hosts+ "https://google.example.com/search?q=firecrawl",+ "https://notgoogle.com/search?q=firecrawl",+ "https://example.com/search?q=firecrawl",+ // Not a URL+ "not a url",+ "ftp://www.google.com/search?q=firecrawl",+ ])("ignores %s", url => {+ expect(detectSerpPage(url)).toBeNull();+ });+});++describe("serpScrapeWarning", () => {+ it("names the engine and points to /search", () => {+ const warning = serpScrapeWarning("Google");+ expect(warning).toContain("Google search results page");+ expect(warning).toContain("/search endpoint");+ expect(warning).toContain("https://docs.firecrawl.dev/features/search");+ });+});++describe("serpScrapeWarningEnabled", () => {+ beforeEach(() => {+ mockConfig.SERP_SCRAPE_WARNING = undefined;+ mockConfig.SERP_SCRAPE_WARNING_TEAM_IDS = undefined;+ mockConfig.SERP_SCRAPE_WARNING_ROLLOUT_PERCENT = 0;+ });++ it("is off by default", () => {+ expect(serpScrapeWarningEnabled("team-a")).toBe(false);+ });++ it("is on for every team when SERP_SCRAPE_WARNING is set", () => {+ mockConfig.SERP_SCRAPE_WARNING = true;+ expect(serpScrapeWarningEnabled("team-a")).toBe(true);+ });++ it("is on only for listed teams", () => {+ mockConfig.SERP_SCRAPE_WARNING_TEAM_IDS = ["team-a"];+ expect(serpScrapeWarningEnabled("team-a")).toBe(true);+ expect(serpScrapeWarningEnabled("team-b")).toBe(false);+ });++ it("follows the rollout percent", () => {+ mockConfig.SERP_SCRAPE_WARNING_ROLLOUT_PERCENT = 100;+ expect(serpScrapeWarningEnabled("team-a")).toBe(true);+ });+});diff --git a/apps/api/src/lib/serp-url.ts b/apps/api/src/lib/serp-url.tsnew file mode 100644index 000000000..0c7b26bf9--- /dev/null+++ b/apps/api/src/lib/serp-url.ts@@ -0,0 +1,115 @@+import { config } from "../config";+import { sampled } from "./rollout";+import { parseHostname } from "./url-utils";++type SerpEngine =+ | "Google"+ | "Bing"+ | "DuckDuckGo"+ | "Brave"+ | "Yahoo"+ | "Baidu"+ | "Yandex"+ | "Ecosia"+ | "Startpage";++type SerpPage = { engine: SerpEngine; query: string };++const MAX_QUERY_CHARS = 200;++function queryParam(params: URLSearchParams, ...names: string[]): string {+ for (const name of names) {+ const value = (params.get(name) ?? "").trim();+ if (value) return value.slice(0, MAX_QUERY_CHARS);+ }+ return "";+}++function engineAndQuery(url: URL): SerpPage | null {+ const host = url.hostname.toLowerCase();+ const parsedHost = parseHostname(host);+ const domain = parsedHost.domain ?? "";+ const domainWithoutSuffix = parsedHost.domainWithoutSuffix ?? "";+ const subdomain = parsedHost.subdomain ?? "";+ const path = url.pathname.replace(/\/+$/, "") || "/";+ const params = url.searchParams;+ const bareOrWww = subdomain === "" || subdomain === "www";+ const page = (engine: SerpEngine, matches: boolean, ...names: string[]) => {+ const query = matches ? queryParam(params, ...names) : "";+ return query ? { engine, query } : null;+ };++ if (domainWithoutSuffix === "google" && bareOrWww) {+ return page("Google", path === "/search", "q");+ }+ if (domain === "bing.com") {+ return page("Bing", path === "/search", "q");+ }+ if (domain === "duckduckgo.com") {+ return page(+ "DuckDuckGo",+ path === "/" || path === "/html" || path === "/lite",+ "q",+ );+ }+ if (host === "search.brave.com") {+ return page("Brave", path === "/search", "q");+ }+ if (+ domainWithoutSuffix === "yahoo" &&+ (subdomain === "search" || subdomain.endsWith(".search"))+ ) {+ // Yahoo appends tracking after a semicolon: /search;_ylt=...+ return page("Yahoo", /^\/search(?:[/;]|$)/.test(path), "p", "q");+ }+ if (domain === "baidu.com" && bareOrWww) {+ return page("Baidu", path === "/s", "wd", "word");+ }+ if (domainWithoutSuffix === "yandex" && bareOrWww) {+ return page("Yandex", path === "/search", "text");+ }+ if (domain === "ecosia.org") {+ return page("Ecosia", path === "/search", "q");+ }+ if (domain === "startpage.com") {+ return page(+ "Startpage",+ path === "/do/search" || path === "/sp/search",+ "q",+ "query",+ );+ }+ return null;+}++/**+ * Returns the search engine and query when the URL is a results page for a+ * query (google.*\/search?q=, bing.com/search?q=, ...), otherwise null. Other+ * pages on those domains (Maps, Shopping product pages, the home page) are not+ * results pages and return null.+ */+export function detectSerpPage(url: string): SerpPage | null {+ let parsed: URL;+ try {+ parsed = new URL(url);+ } catch {+ return null;+ }+ if (parsed.protocol !== "http:" && parsed.protocol !== "https:") return null;+ return engineAndQuery(parsed);+}++export function serpScrapeWarning(engine: SerpEngine): string {+ return `This URL is a ${engine} search results page. Firecrawl's /search endpoint returns ranked results for a query without the page's ads and layout, and can scrape each result in the same call: https://docs.firecrawl.dev/features/search`;+}++/**+ * Whether a team sees the SERP scrape warning. SERP_SCRAPE_WARNING turns it on+ * for every team, SERP_SCRAPE_WARNING_TEAM_IDS for listed teams, and+ * SERP_SCRAPE_WARNING_ROLLOUT_PERCENT for a stable share of teams.+ */+export function serpScrapeWarningEnabled(teamId: string): boolean {+ if (config.SERP_SCRAPE_WARNING) return true;+ if (config.SERP_SCRAPE_WARNING_TEAM_IDS?.includes(teamId)) return true;+ return sampled(`team:${teamId}`, config.SERP_SCRAPE_WARNING_ROLLOUT_PERCENT);+}diff --git a/apps/api/src/services/worker/scrape-worker.ts b/apps/api/src/services/worker/scrape-worker.tsindex bfb7d562c..52c41891b 100644--- a/apps/api/src/services/worker/scrape-worker.ts+++ b/apps/api/src/services/worker/scrape-worker.ts@@ -47,6 +47,11 @@ import {addScrapeJobs,} from "../queue-jobs";import { parseHostname } from "../../lib/url-utils";+import {+ detectSerpPage,+ serpScrapeWarning,+ serpScrapeWarningEnabled,+} from "../../lib/serp-url";import { getJobPriority } from "../../lib/job-priority";import { Document, scrapeOptions, TeamFlags } from "../../controllers/v2/types";import { hasFormatOfType } from "../../lib/format-utils";@@ -539,6 +544,26 @@ async function processJob(job: NuQJob<ScrapeJobSingleUrls>) {(doc.warning ? " " + doc.warning : "");}+ const billingEndpoint = job.data.billing?.endpoint;+ if (billingEndpoint === "scrape" || billingEndpoint === "batch_scrape") {+ const serpPage = detectSerpPage(job.data.url);+ if (serpPage !== null) {+ // Logged whether or not the warning is shown, so the rollout can be+ // compared with teams outside it.+ const warned = serpScrapeWarningEnabled(job.data.team_id);+ logger.info("SERP scrape", {+ serpEngine: serpPage.engine,+ serpWarningShown: warned,+ billingEndpoint,+ });+ if (warned) {+ doc.warning =+ serpScrapeWarning(serpPage.engine) ++ (doc.warning ? " " + doc.warning : "");+ }+ }+ }+const data = {success: true,result: {