Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): serve government web searches through the Government Index (#5672)
Ankit Bahuguna346045851ba7
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 2b709ceb1..fdfce8858 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -129,6 +129,15 @@ const configSchema = z.object({),RESEARCH_PROXY_URL: z.string().url().optional(),SEARCH_PLATFORM_URL: z.string().url().optional(),+ // When true, a plain web search that the search platform classifies as+ // government is served through the Government Index. It needs+ // SEARCH_PLATFORM_URL. Unset or empty, no search is routed.+ GOV_ROUTING_ENABLED: emptyStringAsDefault(z.stringbool().default(false)),+ // The longest a plain web search waits for the routing decision before the+ // regular web search starts; a decision that takes longer routes nothing.+ GOV_ROUTING_DECISION_TIMEOUT_MS: emptyStringAsDefault(+ z.coerce.number().int().positive().default(100),+ ),RESEARCH_KEYLESS_DISABLED: researchKeylessDisabled,// Per-team Government Index daily limits as teamId:limit,teamId:limit. A// listed team's limit replaces its plan limit.@@ -897,6 +906,14 @@ const validatedConfigSchema = configSchema.superRefine((value, context) => {});}}+ if (value.GOV_ROUTING_ENABLED && !value.SEARCH_PLATFORM_URL) {+ context.addIssue({+ code: "custom",+ path: ["GOV_ROUTING_ENABLED"],+ message:+ "GOV_ROUTING_ENABLED needs SEARCH_PLATFORM_URL: set it, or turn routing off",+ });+ }for (const error of getMcpActionLogConfigErrors(value)) {context.addIssue({code: "custom",diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.tsindex 0cd0cfd27..af095bdb2 100644--- a/apps/api/src/lib/research-upstream.ts+++ b/apps/api/src/lib/research-upstream.ts@@ -82,6 +82,39 @@ export function usOnlySkippedWarning(headers: {: undefined;}+/** POSTs a JSON body to the search platform, or returns null when none is configured. */+async function postSearchPlatform(+ path: string,+ body: unknown,+ headers: Record<string, string>,+ timeoutMs: number,+) {+ const base = config.SEARCH_PLATFORM_URL;+ if (!base) return null;++ return fetch(base.replace(/\/+$/, "") + path, {+ method: "POST",+ headers: { ...headers, "content-type": "application/json" },+ body: JSON.stringify(body),+ signal: AbortSignal.timeout(timeoutMs),+ dispatcher,+ });+}++/** The search platform's routing decision for one query. */+export async function fetchQueryFeatures(options: {+ query: string;+ headers: Record<string, string>;+ timeoutMs: number;+}) {+ return postSearchPlatform(+ "/api/v1/query-features",+ { queries: [options.query], features: ["vertical"] },+ options.headers,+ options.timeoutMs,+ );+}+export async function fetchGovUpstream(options: {query: string;k?: number;@@ -93,13 +126,9 @@ export async function fetchGovUpstream(options: {headers: Record<string, string>;timeoutMs: number;}) {- const base = config.SEARCH_PLATFORM_URL;- if (!base) return null;-- return fetch(base.replace(/\/+$/, "") + "/api/v1/gov-search", {- method: "POST",- headers: { ...options.headers, "content-type": "application/json" },- body: JSON.stringify({+ return postSearchPlatform(+ "/api/v1/gov-search",+ {query: options.query,top_k: options.k,jurisdiction: options.jurisdiction,@@ -107,8 +136,23 @@ export async function fetchGovUpstream(options: {court_type: options.courtType,cites: options.cites,domain: options.domain,- }),- signal: AbortSignal.timeout(options.timeoutMs),- dispatcher,- });+ },+ options.headers,+ options.timeoutMs,+ );+}++/** The search platform's answer to a plain web search routed to it. */+export async function fetchWebSearchUpstream(options: {+ query: string;+ k: number;+ headers: Record<string, string>;+ timeoutMs: number;+}) {+ return postSearchPlatform(+ "/api/v1/web-search",+ { query: options.query, top_k: options.k },+ options.headers,+ options.timeoutMs,+ );}diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsindex c66feea59..83756243a 100644--- a/apps/api/src/search/execute.test.ts+++ b/apps/api/src/search/execute.test.ts@@ -2,6 +2,7 @@ const mocks = vi.hoisted(() => ({search: vi.fn(),searchDeveloperCategory: vi.fn(),searchGovCategory: vi.fn(),+ searchGovIfRouted: vi.fn(),checkUrlsAgainstThreatPolicy: vi.fn(),discoverTools: vi.fn(),discoverSkills: vi.fn(),@@ -48,6 +49,8 @@ vi.mock("./gov", () => ({: category.type === "gov",),searchGovCategory: mocks.searchGovCategory,+ searchGovIfRouted: mocks.searchGovIfRouted,+ MAX_ROUTED_LIMIT: 10,}));vi.mock("./scrape", () => ({getItemsToScrape: vi.fn(() => []),@@ -121,6 +124,7 @@ function options(categories: Array<{ type: string }>) {beforeEach(() => {vi.clearAllMocks();mocks.searchDeveloperCategory.mockResolvedValue([developerResult]);+ mocks.searchGovIfRouted.mockResolvedValue(null);mocks.discoverTools.mockResolvedValue({status: "available",items: [{ id: "fred/series/observations" }],@@ -893,6 +897,116 @@ describe("executeSearch gov category", () => {});});+describe("executeSearch gov routing", () => {+ const govResult = {+ url: "https://www.dol.gov/agencies/whd/fmla",+ title: "FMLA",+ description: "Family and Medical Leave Act",+ position: 1,+ };+ const webResult = {+ url: "https://example.com/",+ title: "Example",+ description: "",+ };++ it("serves a routed plain web search from the Government Index, skips the regular web search, and bills it as a web search", async () => {+ mocks.search.mockResolvedValue({ web: [webResult] });+ mocks.searchGovIfRouted.mockResolvedValue([govResult]);++ const result = await executeSearch(options([]), context, logger);++ expect(mocks.searchGovIfRouted).toHaveBeenCalledWith(+ {+ query: "retries",+ limit: 10,+ teamId: "team-1",+ timeout: 1_000,+ zeroDataRetention: false,+ },+ "job-1",+ logger,+ );+ expect(mocks.searchGovCategory).not.toHaveBeenCalled();+ expect(mocks.search).not.toHaveBeenCalled();+ expect(result.response).toEqual({ web: [govResult] });+ expect(result.searchCredits).toBe(2);+ });++ it("starts the regular web search only after the routing decision and serves it when the search is not routed", async () => {+ mocks.search.mockResolvedValue({ web: [webResult] });+ let webSearchCallsWhenRouting = -1;+ mocks.searchGovIfRouted.mockImplementation(async () => {+ webSearchCallsWhenRouting = mocks.search.mock.calls.length;+ return null;+ });++ const result = await executeSearch(options([]), context, logger);++ expect(webSearchCallsWhenRouting).toBe(0);+ expect(mocks.search).toHaveBeenCalledTimes(1);+ expect(result.response.web?.map(item => item.url)).toEqual([+ "https://example.com/",+ ]);+ });++ it.each([+ ["a category", { categories: [{ type: "pdf" }] }],+ ["another source", { sources: [{ type: "web" }, { type: "news" }] }],+ ["included domains", { includeDomains: ["example.com"] }],+ ["excluded domains", { excludeDomains: ["example.com"] }],+ ["a time range", { tbs: "qdr:d" }],+ ["a result filter", { filter: "0" }],+ ["a country other than the US", { country: "gb" }],+ ["a site: operator", { query: "leave site:dol.gov" }],+ [+ "an operator the parser leaves in the text",+ { query: "fmla filetype:pdf" },+ ],+ ["an OR between terms", { query: "fmla OR overtime" }],+ ["a quoted phrase", { query: '"family leave" eligibility' }],+ ["a language other than English", { lang: "es" }],+ ["a location", { location: "Toronto, Ontario" }],+ ["a page larger than 10 results", { limit: 11 }],+ ["a blank query", { query: " " }],+ ])("never routes a search with %s", async (_case, overrides) => {+ mocks.search.mockResolvedValue({ web: [] });++ await executeSearch({ ...options([]), ...overrides }, context, logger);++ expect(mocks.searchGovIfRouted).not.toHaveBeenCalled();+ });++ it.each([+ ["a zero data retention team", {}, { zeroDataRetention: true }],+ ["an enterprise zdr search", { enterprise: ["zdr"] }, {}],+ ["an enterprise anon search", { enterprise: ["anon"] }, {}],+ ])("never routes %s", async (_case, optionOverrides, contextOverrides) => {+ mocks.search.mockResolvedValue({ web: [] });++ await executeSearch(+ { ...options([]), ...optionOverrides },+ { ...context, ...contextOverrides } as any,+ logger,+ );++ expect(mocks.searchGovIfRouted).not.toHaveBeenCalled();+ expect(mocks.search).toHaveBeenCalledTimes(1);+ });++ it("never routes a US-only team", async () => {+ mocks.search.mockResolvedValue({ web: [] });++ await executeSearch(+ options([]),+ { ...context, flags: { usOnly: true } } as any,+ logger,+ );++ expect(mocks.searchGovIfRouted).not.toHaveBeenCalled();+ });+});+describe("executeSearch query operators", () => {const web = (url: string, title = "Result") => ({url,diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 882088719..78cb24afb 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -39,12 +39,18 @@ import {calculateScrapeCredits,} from "./scrape";import { searchDeveloperCategory, wantsDeveloperCategory } from "./developer";-import { searchGovCategory, wantsGovCategory } from "./gov";+import {+ MAX_ROUTED_LIMIT,+ searchGovCategory,+ searchGovIfRouted,+ wantsGovCategory,+} from "./gov";import { removeExplicitResults } from "./safe-search";import {matchesQueryOperators,planOperatorQuery,siteHostnames,+ usesSearchSyntax,} from "./operators";import {highlightsEnvReady,@@ -340,29 +346,51 @@ export async function executeSearch(: wantsGovCategory(categories)? searchGovCategory: null;- const indexResultsPromise = indexCategorySearch- ? indexCategorySearch(- {- query,- limit,- teamId,- timeout: options.timeout,- zeroDataRetention:- zeroDataRetention === true ||- options.enterprise?.some(- mode => mode === "zdr" || mode === "anon",- ) === true,- ...(usOnly ? { usOnly, warnings: usOnlyWarnings } : {}),- },- logger,- )+ const indexSearchOptions = {+ query,+ limit,+ teamId,+ timeout: options.timeout,+ zeroDataRetention:+ zeroDataRetention === true ||+ options.enterprise?.some(mode => mode === "zdr" || mode === "anon") ===+ true,+ ...(usOnly ? { usOnly, warnings: usOnlyWarnings } : {}),+ };+ // A plain US web search that the search platform classifies as government+ // is served through the Government Index. Zero data retention and+ // anonymous searches are never routed, as they skip every other auxiliary+ // service here, and neither is a page larger than MAX_ROUTED_LIMIT.+ const plainWebSearch =+ indexCategorySearch === null &&+ !indexSearchOptions.zeroDataRetention &&+ !categories?.length &&+ searchTypes.length === 1 &&+ searchTypes[0] === "web" &&+ !options.includeDomains?.length &&+ !options.excludeDomains?.length &&+ !options.tbs &&+ !options.filter &&+ (options.country ?? "us").toLowerCase() === "us" &&+ !options.location &&+ (options.lang ?? "en").toLowerCase() === "en" &&+ !usOnly &&+ query.trim().length > 0 &&+ limit <= MAX_ROUTED_LIMIT &&+ !usesSearchSyntax(query);+ // A plain web search takes the routing decision before the regular web+ // search, so a routed search never starts it. An unrouted one, or a routed+ // one whose platform search fails, starts the regular web search after the+ // decision.+ const routedResults = plainWebSearch+ ? await searchGovIfRouted(indexSearchOptions, context.jobId, logger): null;-- const searchResponse =- ((wantsTools || wantsSkills) && !searchTypes.length) ||- indexResultsPromise !== null- ? ({} as SearchV2Response)- : ((await search({+ const webSearchPromise =+ routedResults !== null ||+ indexCategorySearch !== null ||+ ((wantsTools || wantsSkills) && !searchTypes.length)+ ? null+ : search({query: searchQuery,logger,requestId: context.requestId,@@ -386,7 +414,19 @@ export async function executeSearch(...operators.excludedSites.filter(site => !site.includes("/")),]),usOnly,- })) as SearchV2Response);+ });+ const indexResultsPromise = indexCategorySearch+ ? indexCategorySearch(indexSearchOptions, logger)+ : null;++ // Routed results take the regular web search's place as the web results, so+ // threat protection, safe search, categories, highlights, scraping and+ // billing treat them as a web search's.+ const searchResponse: SearchV2Response = routedResults+ ? { web: routedResults }+ : webSearchPromise === null || indexResultsPromise !== null+ ? ({} as SearchV2Response)+ : ((await webSearchPromise) as SearchV2Response);const keepOperatorMatches = <T extends { url?: string; title?: string; position?: number },>(diff --git a/apps/api/src/search/gov.test.ts b/apps/api/src/search/gov.test.tsindex 50ef19664..95e88cf7e 100644--- a/apps/api/src/search/gov.test.ts+++ b/apps/api/src/search/gov.test.ts@@ -1,16 +1,30 @@-vi.mock("undici", () => ({- Agent: class {},+// Hoisted so a module re-imported after vi.resetModules shares them.+const shared = vi.hoisted(() => ({fetch: vi.fn(),+ config: {+ SEARCH_PLATFORM_URL: "https://platform.test/",+ GOV_ROUTING_ENABLED: true,+ GOV_ROUTING_DECISION_TIMEOUT_MS: 100,+ },+ rootLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },}));-vi.mock("../config", () => ({- config: { SEARCH_PLATFORM_URL: "https://platform.test/" },+vi.mock("undici", () => ({+ Agent: class {},+ fetch: shared.fetch,}));-import { fetch } from "undici";-import { searchGovCategory } from "./gov";+vi.mock("../config", () => ({ config: shared.config }));-const fetchMock = vi.mocked(fetch);+vi.mock("../lib/logger", () => ({ logger: shared.rootLogger }));++import {+ searchGovCategory,+ type searchGovIfRouted as routedSearch,+} from "./gov";++const fetchMock = shared.fetch;+const config = shared.config;const logger = {info: vi.fn(),@@ -167,3 +181,397 @@ describe("searchGovCategory", () => {).resolves.toEqual([]);});});++describe("searchGovIfRouted", () => {+ const search = {+ query: "fmla leave eligibility",+ limit: 1,+ teamId: "t1",+ timeout: 500,+ zeroDataRetention: false,+ };+ const vertical = (fields: Record<string, unknown>) =>+ upstreamOk({+ results: [+ {+ vertical: fields,+ },+ ],+ });+ const routedToGov = () =>+ vertical({ search_category_routed: ["GOVERNMENT"] });+ const govHit = upstreamOk({+ success: true,+ data: {+ web: [+ {+ url: "https://www.dol.gov/agencies/whd/fmla",+ title: "FMLA",+ description: "Family and Medical Leave Act",+ },+ ],+ },+ });+ const unavailable = () =>+ ({ ok: false, status: 503, body: { cancel: async () => {} } }) as any;+ const routingLines = () =>+ shared.rootLogger.info.mock.calls+ .filter(([message]) => message === "Search gov routing")+ .map(([, fields]) => fields);+ const outcomes = () => routingLines().map(line => line.outcome);++ // Routing pauses after repeated failures, and that state lives in the module,+ // so every test routes through a fresh copy of it.+ let searchGovIfRouted: typeof routedSearch;+ beforeEach(async () => {+ vi.resetModules();+ ({ searchGovIfRouted } = await import("./gov.js"));+ });++ afterEach(() => {+ config.GOV_ROUTING_ENABLED = true;+ vi.useRealTimers();+ fetchMock.mockReset();+ });++ it("serves the search platform's web search when it routes the query to GOVERNMENT", async () => {+ fetchMock+ .mockResolvedValueOnce(routedToGov())+ .mockResolvedValueOnce(govHit);++ const results = await searchGovIfRouted(search, "s1", logger);++ const [url, init] = fetchMock.mock.calls[0] as any[];+ expect(url).toBe("https://platform.test/api/v1/query-features");+ expect(JSON.parse(init.body)).toEqual({+ queries: ["fmla leave eligibility"],+ features: ["vertical"],+ });+ expect(init.headers["firecrawl-zero-data-retention"]).toBe("false");+ const [webUrl, webInit] = fetchMock.mock.calls[1] as any[];+ expect(webUrl).toBe("https://platform.test/api/v1/web-search");+ expect(JSON.parse(webInit.body)).toEqual({+ query: "fmla leave eligibility",+ top_k: 2,+ });+ expect(results).toEqual([+ {+ url: "https://www.dol.gov/agencies/whd/fmla",+ title: "FMLA",+ description: "Family and Medical Leave Act",+ position: 1,+ },+ ]);+ });++ it("logs one shape-only routing line per search, joinable to the search by its id", async () => {+ fetchMock+ .mockResolvedValueOnce(routedToGov())+ .mockResolvedValueOnce(govHit);++ await searchGovIfRouted(search, "s1", logger);++ expect(routingLines()).toEqual([+ {+ event: "search_gov_routing",+ searchId: "s1",+ teamId: "t1",+ outcome: "routed_served",+ limit: 1,+ decisionMs: expect.any(Number),+ webSearchMs: expect.any(Number),+ results: 1,+ },+ ]);+ expect(JSON.stringify(routingLines())).not.toContain("fmla");+ });++ it("keeps a query the search platform does not route on the web", async () => {+ fetchMock.mockResolvedValueOnce(vertical({ search_category_routed: [] }));++ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();+ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(outcomes()).toEqual(["not_routed"]);+ });++ it.each([+ ["answers 503", unavailable()],+ ["answers without search_category_routed", vertical({})],+ [+ "answers search_category_routed as a string",+ vertical({ search_category_routed: "GOVERNMENT" }),+ ],+ [+ "answers search_category_routed with a non-string entry",+ vertical({ search_category_routed: ["GOVERNMENT", 1] }),+ ],+ ["answers without results", upstreamOk({})],+ ])(+ "keeps the query on the web when the routing decision %s",+ async (_case, response) => {+ fetchMock.mockResolvedValueOnce(response as any);++ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();+ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(outcomes()).toEqual(["decision_failed"]);+ },+ );++ it("keeps the query on the web when the routing decision throws", async () => {+ fetchMock.mockRejectedValueOnce(new Error("timeout"));++ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();+ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(outcomes()).toEqual(["decision_failed"]);+ });++ it("asks nothing and logs nothing when routing is off", async () => {+ config.GOV_ROUTING_ENABLED = false;++ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();+ expect(fetchMock).not.toHaveBeenCalled();+ expect(routingLines()).toEqual([]);+ });++ it.each([+ [+ "answers 404",+ { ok: false, status: 404, body: { cancel: async () => {} } },+ ],+ [+ "answers a body that is not JSON",+ {+ ok: true,+ status: 200,+ json: async () => {+ throw new SyntaxError("Unexpected token <");+ },+ },+ ],+ ["answers without data.web", upstreamOk({ success: true })],+ ])(+ "falls back to the regular web search when the search platform's web search %s",+ async (_case, response) => {+ fetchMock+ .mockResolvedValueOnce(routedToGov())+ .mockResolvedValueOnce(response as any);++ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();+ expect(fetchMock).toHaveBeenCalledTimes(2);+ expect(outcomes()).toEqual(["routed_failed"]);+ },+ );++ it("falls back to the regular web search when a routed query finds nothing", async () => {+ fetchMock+ .mockResolvedValueOnce(routedToGov())+ .mockResolvedValueOnce(upstreamOk({ success: true, data: { web: [] } }));++ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();+ expect(fetchMock).toHaveBeenCalledTimes(2);+ expect(outcomes()).toEqual(["routed_empty"]);+ });++ /**+ * The regular web search serves a page shorter than asked as it is, so a routed answer+ * must too instead of running a second search.+ */+ it("serves a routed answer that holds fewer results than asked", async () => {+ fetchMock+ .mockResolvedValueOnce(routedToGov())+ .mockResolvedValueOnce(govHit);++ const results = await searchGovIfRouted(+ { ...search, limit: 2 },+ "s1",+ logger,+ );++ expect(results?.map(result => result.url)).toEqual([+ "https://www.dol.gov/agencies/whd/fmla",+ ]);+ expect(fetchMock).toHaveBeenCalledTimes(2);+ expect(routingLines()[0]).toMatchObject({+ outcome: "routed_served",+ results: 1,+ limit: 2,+ });+ });++ /**+ * Safe search and threat protection run on routed results later and can+ * remove some, so a routed search keeps up to twice the page, as the regular+ * web search does, and the page is cut to size after filtering.+ */+ it("keeps up to twice the page from a routed answer", async () => {+ const hit = (n: number) => ({+ url: `https://www.dol.gov/${n}`,+ title: `Page ${n}`,+ description: "",+ });+ fetchMock+ .mockResolvedValueOnce(routedToGov())+ .mockResolvedValueOnce(+ upstreamOk({ success: true, data: { web: [hit(1), hit(2), hit(3)] } }),+ );++ const results = await searchGovIfRouted(search, "s1", logger);++ expect(results?.map(result => result.url)).toEqual([+ "https://www.dol.gov/1",+ "https://www.dol.gov/2",+ ]);+ });++ /**+ * While the search platform is down, every routed search would wait out its+ * timeout before the regular web search. After five failed decisions in a+ * row the process stops asking for a minute, so searches go straight to the+ * regular web search, and asks again once the minute is up.+ */+ it("pauses routing for a minute after five failed decisions in a row", async () => {+ vi.useFakeTimers({ toFake: ["Date"] });+ fetchMock.mockRejectedValue(new Error("connect ECONNREFUSED"));++ for (let attempt = 0; attempt < 6; attempt++) {+ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();+ }++ expect(fetchMock).toHaveBeenCalledTimes(5);+ expect(outcomes()).toEqual([...Array(5).fill("decision_failed"), "paused"]);+ expect(shared.rootLogger.warn).toHaveBeenCalledWith(+ "Gov routing paused after repeated search platform failures",+ expect.objectContaining({ call: "decision" }),+ );++ vi.setSystemTime(Date.now() + 60_001);+ fetchMock.mockReset();+ fetchMock.mockResolvedValueOnce(vertical({ search_category_routed: [] }));+ await searchGovIfRouted(search, "s1", logger);+ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(outcomes().at(-1)).toBe("not_routed");+ });++ /**+ * A decision that answers must not clear the web search's failures:+ * otherwise a web search that is down would never reach five failures in a+ * row while decisions keep answering, and routed searches would keep+ * waiting on it.+ */+ it.each([+ ["answer 503", unavailable],+ ["answer 200 without data.web", () => upstreamOk({ success: true })],+ ])(+ "pauses on five web searches in a row that %s, whatever the decisions between them",+ async (_case, broken) => {+ for (let attempt = 0; attempt < 5; attempt++) {+ fetchMock+ .mockResolvedValueOnce(vertical({ search_category_routed: [] }))+ .mockResolvedValueOnce(routedToGov())+ .mockResolvedValueOnce(broken());+ await searchGovIfRouted(search, "s1", logger);+ await searchGovIfRouted(search, "s1", logger);+ }+ const asked = fetchMock.mock.calls.length;++ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();++ expect(fetchMock).toHaveBeenCalledTimes(asked);+ expect(outcomes().at(-1)).toBe("paused");+ expect(shared.rootLogger.warn).toHaveBeenCalledWith(+ "Gov routing paused after repeated search platform failures",+ expect.objectContaining({ call: "webSearch" }),+ );+ },+ );++ it("starts the failure count over after the platform answers", async () => {+ for (let attempt = 0; attempt < 4; attempt++) {+ fetchMock.mockRejectedValueOnce(new Error("timeout"));+ await searchGovIfRouted(search, "s1", logger);+ }+ fetchMock.mockResolvedValueOnce(vertical({ search_category_routed: [] }));+ await searchGovIfRouted(search, "s1", logger);+ fetchMock.mockRejectedValueOnce(new Error("timeout"));+ await searchGovIfRouted(search, "s1", logger);++ expect(outcomes()).not.toContain("paused");+ expect(fetchMock).toHaveBeenCalledTimes(6);+ });++ /**+ * The pause is shared by every team on the process, so a caller must not be+ * able to trip it: a refused query says nothing about the platform's health.+ */+ it("does not pause on queries the search platform refuses", async () => {+ for (let attempt = 0; attempt < 5; attempt++) {+ fetchMock.mockResolvedValueOnce({+ ok: false,+ status: 400,+ body: { cancel: async () => {} },+ } as any);+ await searchGovIfRouted(search, "s1", logger);+ }+ fetchMock.mockResolvedValueOnce(vertical({ search_category_routed: [] }));++ await searchGovIfRouted(search, "s1", logger);++ expect(fetchMock).toHaveBeenCalledTimes(6);+ expect(outcomes()).toEqual([+ ...Array(5).fill("decision_failed"),+ "not_routed",+ ]);+ });++ /**+ * A routed search may take at most half the time left in the request, so the+ * regular web search keeps time to run when the route fails. A timeout set by+ * that deadline comes from the caller's own short `timeout`, so it must not+ * count toward the pause; a timeout at the routed search's own limit does.+ */+ it.each([+ ["the request's own deadline", 500, "not_routed"],+ ["the routed search's own limit", 60_000, "paused"],+ ])(+ "on five routed searches in a row timed out by %s (timeout %i), the next search is %s",+ async (_case, timeout, next) => {+ const timedOut = () =>+ new DOMException(+ "The operation was aborted due to timeout",+ "TimeoutError",+ );+ for (let attempt = 0; attempt < 5; attempt++) {+ fetchMock+ .mockResolvedValueOnce(routedToGov())+ .mockRejectedValueOnce(timedOut());+ await searchGovIfRouted({ ...search, timeout }, "s1", logger);+ }+ fetchMock.mockResolvedValueOnce(vertical({ search_category_routed: [] }));++ await searchGovIfRouted({ ...search, timeout }, "s1", logger);++ expect(outcomes()).toEqual([...Array(5).fill("routed_failed"), next]);+ },+ );++ it("starts no routed search for a decision that was in flight when routing paused", async () => {+ let answerDecision!: (response: unknown) => void;+ fetchMock.mockImplementationOnce(+ () =>+ new Promise(resolve => {+ answerDecision = resolve;+ }),+ );+ const inFlight = searchGovIfRouted(search, "s0", logger);+ for (let attempt = 0; attempt < 5; attempt++) {+ fetchMock.mockRejectedValueOnce(new Error("connect ECONNREFUSED"));+ await searchGovIfRouted(search, "s1", logger);+ }++ answerDecision(routedToGov());++ await expect(inFlight).resolves.toBeNull();+ expect(fetchMock).toHaveBeenCalledTimes(6);+ expect(outcomes()).toEqual([...Array(5).fill("decision_failed"), "paused"]);+ });+});diff --git a/apps/api/src/search/gov.ts b/apps/api/src/search/gov.tsindex 2456aa835..43e0d6dd1 100644--- a/apps/api/src/search/gov.ts+++ b/apps/api/src/search/gov.ts@@ -1,13 +1,65 @@import type { Logger } from "winston";import type { WebSearchResult } from "../lib/entities";+import { config } from "../config";import {fetchGovUpstream,+ fetchQueryFeatures,+ fetchWebSearchUpstream,usOnlySkippedWarning,usOnlyUpstreamRefusal,UsOnlyUpstreamRefusalError,} from "../lib/research-upstream";+import { logger as rootLogger } from "../lib/logger";import { hasCategory, type CategoryOption } from "../lib/search-query-builder";+/**+ * The longest a routed search waits for the search platform before falling+ * back to the regular web search, which starts only then.+ */+const ROUTED_SEARCH_TIMEOUT_MS = 4000;++/**+ * The largest page a search may ask for and still be routed. A routed answer+ * may hold no more than this many results, so a larger page stays on the+ * regular web search.+ */+export const MAX_ROUTED_LIMIT = 10;++/**+ * After this many failures in a row of one search platform call, this process+ * stops routing for PAUSE_MS. An outage then delays a handful of searches per+ * process instead of every routed search until routing is switched off.+ */+const PAUSE_AFTER_FAILURES = 5;+const PAUSE_MS = 60_000;++/**+ * Failures in a row of each search platform call on this process. Only a call's+ * own answer resets its count: a decision that routes nothing says nothing+ * about the web search.+ */+const failuresInARow = { decision: 0, webSearch: 0 };+let pausedUntil = 0;++/**+ * What one search platform call says about the platform's health. A refusal+ * of the query (a 4xx) or a call cut short by the request's own deadline says+ * nothing either way, so no caller can pause routing for every team on the+ * process.+ */+type PlatformHealth = "answered" | "failed" | "unknown";++type GovSearchOptions = Parameters<typeof searchGovCategory>[0];++/** What happened to one plain web search's routing; logged once per search. */+type RoutingOutcome =+ | "paused"+ | "decision_failed"+ | "not_routed"+ | "routed_failed"+ | "routed_empty"+ | "routed_served";+export function wantsGovCategory(categories?: CategoryOption[]): boolean {return hasCategory(categories, "gov");}@@ -56,27 +108,265 @@ export async function searchGovCategory(: undefined;if (skipped) options.warnings?.push(skipped);- const body: any = await upstream.json();- const results: any[] = Array.isArray(body?.data?.web) ? body.data.web : [];- return results- .slice(0, options.limit)- .map((result, index) => ({- url: typeof result?.url === "string" ? result.url : "",- title:- typeof result?.title === "string" && result.title.trim().length > 0- ? result.title- : typeof result?.url === "string"- ? result.url- : "",- description:- typeof result?.description === "string" ? result.description : "",- position: index + 1,- category: "gov",- }))- .filter(result => result.url.length > 0);+ return platformResults(await upstream.json(), options.limit).map(+ result => ({ ...result, category: "gov" }),+ );} catch (error) {if (error instanceof UsOnlyUpstreamRefusalError) throw error;logger.warn("Gov category upstream error", { error });return [];}}++/**+ * The search platform's results for a plain web search that it classifies as+ * government, or null when the search stays on the regular web search: routing+ * is off or paused, the decision failed or did not route the query, or the+ * routed search failed or found nothing. An answer with fewer results than+ * asked is served as it is, as a short page of regular web results is.+ *+ * Routing spends the request's own `timeout`, and leaves at least half of it to+ * the regular web search when the route fails.+ *+ * Every search that reaches the routing decision logs one `search_gov_routing`+ * line with its outcome.+ */+export async function searchGovIfRouted(+ options: GovSearchOptions,+ searchId: string,+ logger: Logger,+): Promise<WebSearchResult[] | null> {+ if (!config.GOV_ROUTING_ENABLED) return null;+ const deadline = Date.now() + options.timeout;++ const record = (+ outcome: RoutingOutcome,+ measured: { decisionMs?: number; webSearchMs?: number; results?: number },+ ) =>+ // Root logger: the request logger carries the query, and this line records shape only.+ rootLogger.info("Search gov routing", {+ event: "search_gov_routing",+ searchId,+ teamId: options.teamId,+ outcome,+ limit: options.limit,+ ...measured,+ });++ if (Date.now() < pausedUntil) {+ record("paused", {});+ return null;+ }++ const decisionStarted = Date.now();+ const decision = await searchCategoriesRouted(options, deadline, logger);+ const decisionMs = Date.now() - decisionStarted;+ countCall("decision", decision.health);+ if (decision.categories === null) {+ record("decision_failed", { decisionMs });+ return null;+ }+ if (!decision.categories.includes("GOVERNMENT")) {+ record("not_routed", { decisionMs });+ return null;+ }+ // Routing may have paused while this decision was in flight.+ if (Date.now() < pausedUntil) {+ record("paused", { decisionMs });+ return null;+ }++ const webSearchStarted = Date.now();+ const routed = await searchRoutedWeb(options, deadline, logger);+ const webSearchMs = Date.now() - webSearchStarted;+ countCall("webSearch", routed.health);+ if (routed.results === null) {+ record("routed_failed", { decisionMs, webSearchMs });+ return null;+ }+ if (routed.results.length === 0) {+ record("routed_empty", { decisionMs, webSearchMs, results: 0 });+ return null;+ }+ record("routed_served", {+ decisionMs,+ webSearchMs,+ results: routed.results.length,+ });+ return routed.results;+}++/**+ * Counts what a search platform call said about the platform's health, and+ * pauses routing on this process once one call has failed+ * PAUSE_AFTER_FAILURES times in a row.+ */+function countCall(+ call: keyof typeof failuresInARow,+ health: PlatformHealth,+): void {+ if (health === "unknown") return;+ if (health === "answered") {+ failuresInARow[call] = 0;+ return;+ }+ failuresInARow[call] += 1;+ if (failuresInARow[call] < PAUSE_AFTER_FAILURES) return;+ failuresInARow.decision = 0;+ failuresInARow.webSearch = 0;+ pausedUntil = Date.now() + PAUSE_MS;+ rootLogger.warn(+ "Gov routing paused after repeated search platform failures",+ {+ call,+ failuresInARow: PAUSE_AFTER_FAILURES,+ pauseMs: PAUSE_MS,+ },+ );+}++/**+ * How long a routing call may take: its own limit, or half the time left+ * before the request's deadline when that is shorter, so a failed route+ * leaves the regular web search at least as long as the route took.+ * `deadlineBound` is true when the deadline, not the limit, set the time.+ */+function routingTimeout(limitMs: number, deadline: number) {+ const halfLeft = Math.max(0, Math.floor((deadline - Date.now()) / 2));+ return halfLeft < limitMs+ ? { timeoutMs: halfLeft, deadlineBound: true }+ : { timeoutMs: limitMs, deadlineBound: false };+}++/**+ * The search platform's web search for a routed query, with null results when+ * the request failed, timed out, or was answered with a status other than 2xx+ * or a body without a `data.web` list. It asks for twice the page, as the+ * regular web search does, so results that safe search or threat protection+ * remove still leave a full page. Its results are served as web results, so+ * they carry no gov category.+ */+async function searchRoutedWeb(+ options: GovSearchOptions,+ deadline: number,+ logger: Logger,+): Promise<{ results: WebSearchResult[] | null; health: PlatformHealth }> {+ const { timeoutMs, deadlineBound } = routingTimeout(+ ROUTED_SEARCH_TIMEOUT_MS,+ deadline,+ );+ const k = options.limit * 2;+ try {+ const upstream = await fetchWebSearchUpstream({+ query: options.query,+ k,+ headers: {+ "firecrawl-team-id": options.teamId,+ "firecrawl-zero-data-retention": String(options.zeroDataRetention),+ },+ timeoutMs,+ });+ if (upstream === null) return { results: null, health: "failed" };+ if (!upstream.ok) {+ await upstream.body?.cancel().catch(() => {});+ logger.warn("Routed web search upstream failed", {+ status: upstream.status,+ });+ return {+ results: null,+ health:+ upstream.status >= 400 && upstream.status < 500+ ? "unknown"+ : "failed",+ };+ }+ const body: any = await upstream.json();+ if (!Array.isArray(body?.data?.web)) {+ logger.warn("Routed web search answered without data.web");+ return { results: null, health: "failed" };+ }+ return { results: platformResults(body, k), health: "answered" };+ } catch (error) {+ logger.warn("Routed web search upstream error", { error });+ return {+ results: null,+ health:+ deadlineBound && error?.name === "TimeoutError" ? "unknown" : "failed",+ };+ }+}++/** The `data.web` results of a search platform answer, renumbered from 1. */+function platformResults(body: any, limit: number): WebSearchResult[] {+ const results: any[] = Array.isArray(body?.data?.web) ? body.data.web : [];+ return results+ .slice(0, limit)+ .map((result, index) => ({+ url: typeof result?.url === "string" ? result.url : "",+ title:+ typeof result?.title === "string" && result.title.trim().length > 0+ ? result.title+ : typeof result?.url === "string"+ ? result.url+ : "",+ description:+ typeof result?.description === "string" ? result.description : "",+ position: index + 1,+ }))+ .filter(result => result.url.length > 0);+}++/**+ * The categories the search platform routes the query to, null when the call+ * failed, timed out, or answered without a list of category names.+ */+async function searchCategoriesRouted(+ options: GovSearchOptions,+ deadline: number,+ logger: Logger,+): Promise<{ categories: string[] | null; health: PlatformHealth }> {+ const { timeoutMs, deadlineBound } = routingTimeout(+ config.GOV_ROUTING_DECISION_TIMEOUT_MS,+ deadline,+ );+ try {+ const upstream = await fetchQueryFeatures({+ query: options.query,+ headers: {+ "firecrawl-zero-data-retention": String(options.zeroDataRetention),+ },+ timeoutMs,+ });+ if (upstream === null) return { categories: null, health: "failed" };+ if (!upstream.ok) {+ await upstream.body?.cancel().catch(() => {});+ logger.warn("Routing decision upstream failed", {+ status: upstream.status,+ });+ return {+ categories: null,+ health:+ upstream.status >= 400 && upstream.status < 500+ ? "unknown"+ : "failed",+ };+ }+ const body: any = await upstream.json();+ const routed = body?.results?.[0]?.vertical?.search_category_routed;+ if (+ !Array.isArray(routed) ||+ !routed.every(category => typeof category === "string")+ ) {+ logger.warn("Routing decision answered without search_category_routed");+ return { categories: null, health: "failed" };+ }+ return { categories: routed, health: "answered" };+ } catch (error) {+ logger.warn("Routing decision upstream error", { error });+ return {+ categories: null,+ health:+ deadlineBound && error?.name === "TimeoutError" ? "unknown" : "failed",+ };+ }+}diff --git a/apps/api/src/search/operators.ts b/apps/api/src/search/operators.tsindex 7ff2930b6..ac2cfd375 100644--- a/apps/api/src/search/operators.ts+++ b/apps/api/src/search/operators.ts@@ -54,6 +54,19 @@ function hasUnsupportedOr(query: string): boolean {return ors > siteOrs;}+/**+ * True when the query uses search syntax: a quoted phrase, a `name:value`+ * token such as `site:` or `filetype:`, or an OR or `|` between terms. A+ * routed search sends the query text alone, so it carries none of these.+ */+export function usesSearchSyntax(query: string): boolean {+ if (query.includes('"')) return true;+ return (+ /(?:^|[\s(-])[a-z]+:\S/i.test(query) ||+ /(?:^|[\s(])(?:OR|\|)(?=[\s)(]|$)/.test(query)+ );+}+export function parseQueryOperators(query: string): QueryOperators {const operators = emptyOperators();for (const match of query.matchAll(OPERATOR_TOKEN)) {