Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): serve government web searches through the Government Index (#5672)

Ankit Bahuguna346045851ba7
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 2b709ceb1..fdfce8858 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -129,6 +129,15 @@ const configSchema = z.object({
),
RESEARCH_PROXY_URL: z.string().url().optional(),
SEARCH_PLATFORM_URL: z.string().url().optional(),
+ // When true, a plain web search that the search platform classifies as
+ // government is served through the Government Index. It needs
+ // SEARCH_PLATFORM_URL. Unset or empty, no search is routed.
+ GOV_ROUTING_ENABLED: emptyStringAsDefault(z.stringbool().default(false)),
+ // The longest a plain web search waits for the routing decision before the
+ // regular web search starts; a decision that takes longer routes nothing.
+ GOV_ROUTING_DECISION_TIMEOUT_MS: emptyStringAsDefault(
+ z.coerce.number().int().positive().default(100),
+ ),
RESEARCH_KEYLESS_DISABLED: researchKeylessDisabled,
// Per-team Government Index daily limits as teamId:limit,teamId:limit. A
// listed team's limit replaces its plan limit.
@@ -897,6 +906,14 @@ const validatedConfigSchema = configSchema.superRefine((value, context) => {
});
}
}
+ if (value.GOV_ROUTING_ENABLED && !value.SEARCH_PLATFORM_URL) {
+ context.addIssue({
+ code: "custom",
+ path: ["GOV_ROUTING_ENABLED"],
+ message:
+ "GOV_ROUTING_ENABLED needs SEARCH_PLATFORM_URL: set it, or turn routing off",
+ });
+ }
for (const error of getMcpActionLogConfigErrors(value)) {
context.addIssue({
code: "custom",
diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.ts
index 0cd0cfd27..af095bdb2 100644
--- a/apps/api/src/lib/research-upstream.ts
+++ b/apps/api/src/lib/research-upstream.ts
@@ -82,6 +82,39 @@ export function usOnlySkippedWarning(headers: {
: undefined;
}
+/** POSTs a JSON body to the search platform, or returns null when none is configured. */
+async function postSearchPlatform(
+ path: string,
+ body: unknown,
+ headers: Record<string, string>,
+ timeoutMs: number,
+) {
+ const base = config.SEARCH_PLATFORM_URL;
+ if (!base) return null;
+
+ return fetch(base.replace(/\/+$/, "") + path, {
+ method: "POST",
+ headers: { ...headers, "content-type": "application/json" },
+ body: JSON.stringify(body),
+ signal: AbortSignal.timeout(timeoutMs),
+ dispatcher,
+ });
+}
+
+/** The search platform's routing decision for one query. */
+export async function fetchQueryFeatures(options: {
+ query: string;
+ headers: Record<string, string>;
+ timeoutMs: number;
+}) {
+ return postSearchPlatform(
+ "/api/v1/query-features",
+ { queries: [options.query], features: ["vertical"] },
+ options.headers,
+ options.timeoutMs,
+ );
+}
+
export async function fetchGovUpstream(options: {
query: string;
k?: number;
@@ -93,13 +126,9 @@ export async function fetchGovUpstream(options: {
headers: Record<string, string>;
timeoutMs: number;
}) {
- const base = config.SEARCH_PLATFORM_URL;
- if (!base) return null;
-
- return fetch(base.replace(/\/+$/, "") + "/api/v1/gov-search", {
- method: "POST",
- headers: { ...options.headers, "content-type": "application/json" },
- body: JSON.stringify({
+ return postSearchPlatform(
+ "/api/v1/gov-search",
+ {
query: options.query,
top_k: options.k,
jurisdiction: options.jurisdiction,
@@ -107,8 +136,23 @@ export async function fetchGovUpstream(options: {
court_type: options.courtType,
cites: options.cites,
domain: options.domain,
- }),
- signal: AbortSignal.timeout(options.timeoutMs),
- dispatcher,
- });
+ },
+ options.headers,
+ options.timeoutMs,
+ );
+}
+
+/** The search platform's answer to a plain web search routed to it. */
+export async function fetchWebSearchUpstream(options: {
+ query: string;
+ k: number;
+ headers: Record<string, string>;
+ timeoutMs: number;
+}) {
+ return postSearchPlatform(
+ "/api/v1/web-search",
+ { query: options.query, top_k: options.k },
+ options.headers,
+ options.timeoutMs,
+ );
}
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
index c66feea59..83756243a 100644
--- a/apps/api/src/search/execute.test.ts
+++ b/apps/api/src/search/execute.test.ts
@@ -2,6 +2,7 @@ const mocks = vi.hoisted(() => ({
search: vi.fn(),
searchDeveloperCategory: vi.fn(),
searchGovCategory: vi.fn(),
+ searchGovIfRouted: vi.fn(),
checkUrlsAgainstThreatPolicy: vi.fn(),
discoverTools: vi.fn(),
discoverSkills: vi.fn(),
@@ -48,6 +49,8 @@ vi.mock("./gov", () => ({
: category.type === "gov",
),
searchGovCategory: mocks.searchGovCategory,
+ searchGovIfRouted: mocks.searchGovIfRouted,
+ MAX_ROUTED_LIMIT: 10,
}));
vi.mock("./scrape", () => ({
getItemsToScrape: vi.fn(() => []),
@@ -121,6 +124,7 @@ function options(categories: Array<{ type: string }>) {
beforeEach(() => {
vi.clearAllMocks();
mocks.searchDeveloperCategory.mockResolvedValue([developerResult]);
+ mocks.searchGovIfRouted.mockResolvedValue(null);
mocks.discoverTools.mockResolvedValue({
status: "available",
items: [{ id: "fred/series/observations" }],
@@ -893,6 +897,116 @@ describe("executeSearch gov category", () => {
});
});
+describe("executeSearch gov routing", () => {
+ const govResult = {
+ url: "https://www.dol.gov/agencies/whd/fmla",
+ title: "FMLA",
+ description: "Family and Medical Leave Act",
+ position: 1,
+ };
+ const webResult = {
+ url: "https://example.com/",
+ title: "Example",
+ description: "",
+ };
+
+ it("serves a routed plain web search from the Government Index, skips the regular web search, and bills it as a web search", async () => {
+ mocks.search.mockResolvedValue({ web: [webResult] });
+ mocks.searchGovIfRouted.mockResolvedValue([govResult]);
+
+ const result = await executeSearch(options([]), context, logger);
+
+ expect(mocks.searchGovIfRouted).toHaveBeenCalledWith(
+ {
+ query: "retries",
+ limit: 10,
+ teamId: "team-1",
+ timeout: 1_000,
+ zeroDataRetention: false,
+ },
+ "job-1",
+ logger,
+ );
+ expect(mocks.searchGovCategory).not.toHaveBeenCalled();
+ expect(mocks.search).not.toHaveBeenCalled();
+ expect(result.response).toEqual({ web: [govResult] });
+ expect(result.searchCredits).toBe(2);
+ });
+
+ it("starts the regular web search only after the routing decision and serves it when the search is not routed", async () => {
+ mocks.search.mockResolvedValue({ web: [webResult] });
+ let webSearchCallsWhenRouting = -1;
+ mocks.searchGovIfRouted.mockImplementation(async () => {
+ webSearchCallsWhenRouting = mocks.search.mock.calls.length;
+ return null;
+ });
+
+ const result = await executeSearch(options([]), context, logger);
+
+ expect(webSearchCallsWhenRouting).toBe(0);
+ expect(mocks.search).toHaveBeenCalledTimes(1);
+ expect(result.response.web?.map(item => item.url)).toEqual([
+ "https://example.com/",
+ ]);
+ });
+
+ it.each([
+ ["a category", { categories: [{ type: "pdf" }] }],
+ ["another source", { sources: [{ type: "web" }, { type: "news" }] }],
+ ["included domains", { includeDomains: ["example.com"] }],
+ ["excluded domains", { excludeDomains: ["example.com"] }],
+ ["a time range", { tbs: "qdr:d" }],
+ ["a result filter", { filter: "0" }],
+ ["a country other than the US", { country: "gb" }],
+ ["a site: operator", { query: "leave site:dol.gov" }],
+ [
+ "an operator the parser leaves in the text",
+ { query: "fmla filetype:pdf" },
+ ],
+ ["an OR between terms", { query: "fmla OR overtime" }],
+ ["a quoted phrase", { query: '"family leave" eligibility' }],
+ ["a language other than English", { lang: "es" }],
+ ["a location", { location: "Toronto, Ontario" }],
+ ["a page larger than 10 results", { limit: 11 }],
+ ["a blank query", { query: " " }],
+ ])("never routes a search with %s", async (_case, overrides) => {
+ mocks.search.mockResolvedValue({ web: [] });
+
+ await executeSearch({ ...options([]), ...overrides }, context, logger);
+
+ expect(mocks.searchGovIfRouted).not.toHaveBeenCalled();
+ });
+
+ it.each([
+ ["a zero data retention team", {}, { zeroDataRetention: true }],
+ ["an enterprise zdr search", { enterprise: ["zdr"] }, {}],
+ ["an enterprise anon search", { enterprise: ["anon"] }, {}],
+ ])("never routes %s", async (_case, optionOverrides, contextOverrides) => {
+ mocks.search.mockResolvedValue({ web: [] });
+
+ await executeSearch(
+ { ...options([]), ...optionOverrides },
+ { ...context, ...contextOverrides } as any,
+ logger,
+ );
+
+ expect(mocks.searchGovIfRouted).not.toHaveBeenCalled();
+ expect(mocks.search).toHaveBeenCalledTimes(1);
+ });
+
+ it("never routes a US-only team", async () => {
+ mocks.search.mockResolvedValue({ web: [] });
+
+ await executeSearch(
+ options([]),
+ { ...context, flags: { usOnly: true } } as any,
+ logger,
+ );
+
+ expect(mocks.searchGovIfRouted).not.toHaveBeenCalled();
+ });
+});
+
describe("executeSearch query operators", () => {
const web = (url: string, title = "Result") => ({
url,
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 882088719..78cb24afb 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -39,12 +39,18 @@ import {
calculateScrapeCredits,
} from "./scrape";
import { searchDeveloperCategory, wantsDeveloperCategory } from "./developer";
-import { searchGovCategory, wantsGovCategory } from "./gov";
+import {
+ MAX_ROUTED_LIMIT,
+ searchGovCategory,
+ searchGovIfRouted,
+ wantsGovCategory,
+} from "./gov";
import { removeExplicitResults } from "./safe-search";
import {
matchesQueryOperators,
planOperatorQuery,
siteHostnames,
+ usesSearchSyntax,
} from "./operators";
import {
highlightsEnvReady,
@@ -340,29 +346,51 @@ export async function executeSearch(
: wantsGovCategory(categories)
? searchGovCategory
: null;
- const indexResultsPromise = indexCategorySearch
- ? indexCategorySearch(
- {
- query,
- limit,
- teamId,
- timeout: options.timeout,
- zeroDataRetention:
- zeroDataRetention === true ||
- options.enterprise?.some(
- mode => mode === "zdr" || mode === "anon",
- ) === true,
- ...(usOnly ? { usOnly, warnings: usOnlyWarnings } : {}),
- },
- logger,
- )
+ const indexSearchOptions = {
+ query,
+ limit,
+ teamId,
+ timeout: options.timeout,
+ zeroDataRetention:
+ zeroDataRetention === true ||
+ options.enterprise?.some(mode => mode === "zdr" || mode === "anon") ===
+ true,
+ ...(usOnly ? { usOnly, warnings: usOnlyWarnings } : {}),
+ };
+ // A plain US web search that the search platform classifies as government
+ // is served through the Government Index. Zero data retention and
+ // anonymous searches are never routed, as they skip every other auxiliary
+ // service here, and neither is a page larger than MAX_ROUTED_LIMIT.
+ const plainWebSearch =
+ indexCategorySearch === null &&
+ !indexSearchOptions.zeroDataRetention &&
+ !categories?.length &&
+ searchTypes.length === 1 &&
+ searchTypes[0] === "web" &&
+ !options.includeDomains?.length &&
+ !options.excludeDomains?.length &&
+ !options.tbs &&
+ !options.filter &&
+ (options.country ?? "us").toLowerCase() === "us" &&
+ !options.location &&
+ (options.lang ?? "en").toLowerCase() === "en" &&
+ !usOnly &&
+ query.trim().length > 0 &&
+ limit <= MAX_ROUTED_LIMIT &&
+ !usesSearchSyntax(query);
+ // A plain web search takes the routing decision before the regular web
+ // search, so a routed search never starts it. An unrouted one, or a routed
+ // one whose platform search fails, starts the regular web search after the
+ // decision.
+ const routedResults = plainWebSearch
+ ? await searchGovIfRouted(indexSearchOptions, context.jobId, logger)
: null;
-
- const searchResponse =
- ((wantsTools || wantsSkills) && !searchTypes.length) ||
- indexResultsPromise !== null
- ? ({} as SearchV2Response)
- : ((await search({
+ const webSearchPromise =
+ routedResults !== null ||
+ indexCategorySearch !== null ||
+ ((wantsTools || wantsSkills) && !searchTypes.length)
+ ? null
+ : search({
query: searchQuery,
logger,
requestId: context.requestId,
@@ -386,7 +414,19 @@ export async function executeSearch(
...operators.excludedSites.filter(site => !site.includes("/")),
]),
usOnly,
- })) as SearchV2Response);
+ });
+ const indexResultsPromise = indexCategorySearch
+ ? indexCategorySearch(indexSearchOptions, logger)
+ : null;
+
+ // Routed results take the regular web search's place as the web results, so
+ // threat protection, safe search, categories, highlights, scraping and
+ // billing treat them as a web search's.
+ const searchResponse: SearchV2Response = routedResults
+ ? { web: routedResults }
+ : webSearchPromise === null || indexResultsPromise !== null
+ ? ({} as SearchV2Response)
+ : ((await webSearchPromise) as SearchV2Response);
const keepOperatorMatches = <
T extends { url?: string; title?: string; position?: number },
>(
diff --git a/apps/api/src/search/gov.test.ts b/apps/api/src/search/gov.test.ts
index 50ef19664..95e88cf7e 100644
--- a/apps/api/src/search/gov.test.ts
+++ b/apps/api/src/search/gov.test.ts
@@ -1,16 +1,30 @@
-vi.mock("undici", () => ({
- Agent: class {},
+// Hoisted so a module re-imported after vi.resetModules shares them.
+const shared = vi.hoisted(() => ({
fetch: vi.fn(),
+ config: {
+ SEARCH_PLATFORM_URL: "https://platform.test/",
+ GOV_ROUTING_ENABLED: true,
+ GOV_ROUTING_DECISION_TIMEOUT_MS: 100,
+ },
+ rootLogger: { info: vi.fn(), warn: vi.fn(), error: vi.fn() },
}));
-vi.mock("../config", () => ({
- config: { SEARCH_PLATFORM_URL: "https://platform.test/" },
+vi.mock("undici", () => ({
+ Agent: class {},
+ fetch: shared.fetch,
}));
-import { fetch } from "undici";
-import { searchGovCategory } from "./gov";
+vi.mock("../config", () => ({ config: shared.config }));
-const fetchMock = vi.mocked(fetch);
+vi.mock("../lib/logger", () => ({ logger: shared.rootLogger }));
+
+import {
+ searchGovCategory,
+ type searchGovIfRouted as routedSearch,
+} from "./gov";
+
+const fetchMock = shared.fetch;
+const config = shared.config;
const logger = {
info: vi.fn(),
@@ -167,3 +181,397 @@ describe("searchGovCategory", () => {
).resolves.toEqual([]);
});
});
+
+describe("searchGovIfRouted", () => {
+ const search = {
+ query: "fmla leave eligibility",
+ limit: 1,
+ teamId: "t1",
+ timeout: 500,
+ zeroDataRetention: false,
+ };
+ const vertical = (fields: Record<string, unknown>) =>
+ upstreamOk({
+ results: [
+ {
+ vertical: fields,
+ },
+ ],
+ });
+ const routedToGov = () =>
+ vertical({ search_category_routed: ["GOVERNMENT"] });
+ const govHit = upstreamOk({
+ success: true,
+ data: {
+ web: [
+ {
+ url: "https://www.dol.gov/agencies/whd/fmla",
+ title: "FMLA",
+ description: "Family and Medical Leave Act",
+ },
+ ],
+ },
+ });
+ const unavailable = () =>
+ ({ ok: false, status: 503, body: { cancel: async () => {} } }) as any;
+ const routingLines = () =>
+ shared.rootLogger.info.mock.calls
+ .filter(([message]) => message === "Search gov routing")
+ .map(([, fields]) => fields);
+ const outcomes = () => routingLines().map(line => line.outcome);
+
+ // Routing pauses after repeated failures, and that state lives in the module,
+ // so every test routes through a fresh copy of it.
+ let searchGovIfRouted: typeof routedSearch;
+ beforeEach(async () => {
+ vi.resetModules();
+ ({ searchGovIfRouted } = await import("./gov.js"));
+ });
+
+ afterEach(() => {
+ config.GOV_ROUTING_ENABLED = true;
+ vi.useRealTimers();
+ fetchMock.mockReset();
+ });
+
+ it("serves the search platform's web search when it routes the query to GOVERNMENT", async () => {
+ fetchMock
+ .mockResolvedValueOnce(routedToGov())
+ .mockResolvedValueOnce(govHit);
+
+ const results = await searchGovIfRouted(search, "s1", logger);
+
+ const [url, init] = fetchMock.mock.calls[0] as any[];
+ expect(url).toBe("https://platform.test/api/v1/query-features");
+ expect(JSON.parse(init.body)).toEqual({
+ queries: ["fmla leave eligibility"],
+ features: ["vertical"],
+ });
+ expect(init.headers["firecrawl-zero-data-retention"]).toBe("false");
+ const [webUrl, webInit] = fetchMock.mock.calls[1] as any[];
+ expect(webUrl).toBe("https://platform.test/api/v1/web-search");
+ expect(JSON.parse(webInit.body)).toEqual({
+ query: "fmla leave eligibility",
+ top_k: 2,
+ });
+ expect(results).toEqual([
+ {
+ url: "https://www.dol.gov/agencies/whd/fmla",
+ title: "FMLA",
+ description: "Family and Medical Leave Act",
+ position: 1,
+ },
+ ]);
+ });
+
+ it("logs one shape-only routing line per search, joinable to the search by its id", async () => {
+ fetchMock
+ .mockResolvedValueOnce(routedToGov())
+ .mockResolvedValueOnce(govHit);
+
+ await searchGovIfRouted(search, "s1", logger);
+
+ expect(routingLines()).toEqual([
+ {
+ event: "search_gov_routing",
+ searchId: "s1",
+ teamId: "t1",
+ outcome: "routed_served",
+ limit: 1,
+ decisionMs: expect.any(Number),
+ webSearchMs: expect.any(Number),
+ results: 1,
+ },
+ ]);
+ expect(JSON.stringify(routingLines())).not.toContain("fmla");
+ });
+
+ it("keeps a query the search platform does not route on the web", async () => {
+ fetchMock.mockResolvedValueOnce(vertical({ search_category_routed: [] }));
+
+ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(outcomes()).toEqual(["not_routed"]);
+ });
+
+ it.each([
+ ["answers 503", unavailable()],
+ ["answers without search_category_routed", vertical({})],
+ [
+ "answers search_category_routed as a string",
+ vertical({ search_category_routed: "GOVERNMENT" }),
+ ],
+ [
+ "answers search_category_routed with a non-string entry",
+ vertical({ search_category_routed: ["GOVERNMENT", 1] }),
+ ],
+ ["answers without results", upstreamOk({})],
+ ])(
+ "keeps the query on the web when the routing decision %s",
+ async (_case, response) => {
+ fetchMock.mockResolvedValueOnce(response as any);
+
+ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(outcomes()).toEqual(["decision_failed"]);
+ },
+ );
+
+ it("keeps the query on the web when the routing decision throws", async () => {
+ fetchMock.mockRejectedValueOnce(new Error("timeout"));
+
+ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(outcomes()).toEqual(["decision_failed"]);
+ });
+
+ it("asks nothing and logs nothing when routing is off", async () => {
+ config.GOV_ROUTING_ENABLED = false;
+
+ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();
+ expect(fetchMock).not.toHaveBeenCalled();
+ expect(routingLines()).toEqual([]);
+ });
+
+ it.each([
+ [
+ "answers 404",
+ { ok: false, status: 404, body: { cancel: async () => {} } },
+ ],
+ [
+ "answers a body that is not JSON",
+ {
+ ok: true,
+ status: 200,
+ json: async () => {
+ throw new SyntaxError("Unexpected token <");
+ },
+ },
+ ],
+ ["answers without data.web", upstreamOk({ success: true })],
+ ])(
+ "falls back to the regular web search when the search platform's web search %s",
+ async (_case, response) => {
+ fetchMock
+ .mockResolvedValueOnce(routedToGov())
+ .mockResolvedValueOnce(response as any);
+
+ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ expect(outcomes()).toEqual(["routed_failed"]);
+ },
+ );
+
+ it("falls back to the regular web search when a routed query finds nothing", async () => {
+ fetchMock
+ .mockResolvedValueOnce(routedToGov())
+ .mockResolvedValueOnce(upstreamOk({ success: true, data: { web: [] } }));
+
+ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ expect(outcomes()).toEqual(["routed_empty"]);
+ });
+
+ /**
+ * The regular web search serves a page shorter than asked as it is, so a routed answer
+ * must too instead of running a second search.
+ */
+ it("serves a routed answer that holds fewer results than asked", async () => {
+ fetchMock
+ .mockResolvedValueOnce(routedToGov())
+ .mockResolvedValueOnce(govHit);
+
+ const results = await searchGovIfRouted(
+ { ...search, limit: 2 },
+ "s1",
+ logger,
+ );
+
+ expect(results?.map(result => result.url)).toEqual([
+ "https://www.dol.gov/agencies/whd/fmla",
+ ]);
+ expect(fetchMock).toHaveBeenCalledTimes(2);
+ expect(routingLines()[0]).toMatchObject({
+ outcome: "routed_served",
+ results: 1,
+ limit: 2,
+ });
+ });
+
+ /**
+ * Safe search and threat protection run on routed results later and can
+ * remove some, so a routed search keeps up to twice the page, as the regular
+ * web search does, and the page is cut to size after filtering.
+ */
+ it("keeps up to twice the page from a routed answer", async () => {
+ const hit = (n: number) => ({
+ url: `https://www.dol.gov/${n}`,
+ title: `Page ${n}`,
+ description: "",
+ });
+ fetchMock
+ .mockResolvedValueOnce(routedToGov())
+ .mockResolvedValueOnce(
+ upstreamOk({ success: true, data: { web: [hit(1), hit(2), hit(3)] } }),
+ );
+
+ const results = await searchGovIfRouted(search, "s1", logger);
+
+ expect(results?.map(result => result.url)).toEqual([
+ "https://www.dol.gov/1",
+ "https://www.dol.gov/2",
+ ]);
+ });
+
+ /**
+ * While the search platform is down, every routed search would wait out its
+ * timeout before the regular web search. After five failed decisions in a
+ * row the process stops asking for a minute, so searches go straight to the
+ * regular web search, and asks again once the minute is up.
+ */
+ it("pauses routing for a minute after five failed decisions in a row", async () => {
+ vi.useFakeTimers({ toFake: ["Date"] });
+ fetchMock.mockRejectedValue(new Error("connect ECONNREFUSED"));
+
+ for (let attempt = 0; attempt < 6; attempt++) {
+ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();
+ }
+
+ expect(fetchMock).toHaveBeenCalledTimes(5);
+ expect(outcomes()).toEqual([...Array(5).fill("decision_failed"), "paused"]);
+ expect(shared.rootLogger.warn).toHaveBeenCalledWith(
+ "Gov routing paused after repeated search platform failures",
+ expect.objectContaining({ call: "decision" }),
+ );
+
+ vi.setSystemTime(Date.now() + 60_001);
+ fetchMock.mockReset();
+ fetchMock.mockResolvedValueOnce(vertical({ search_category_routed: [] }));
+ await searchGovIfRouted(search, "s1", logger);
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(outcomes().at(-1)).toBe("not_routed");
+ });
+
+ /**
+ * A decision that answers must not clear the web search's failures:
+ * otherwise a web search that is down would never reach five failures in a
+ * row while decisions keep answering, and routed searches would keep
+ * waiting on it.
+ */
+ it.each([
+ ["answer 503", unavailable],
+ ["answer 200 without data.web", () => upstreamOk({ success: true })],
+ ])(
+ "pauses on five web searches in a row that %s, whatever the decisions between them",
+ async (_case, broken) => {
+ for (let attempt = 0; attempt < 5; attempt++) {
+ fetchMock
+ .mockResolvedValueOnce(vertical({ search_category_routed: [] }))
+ .mockResolvedValueOnce(routedToGov())
+ .mockResolvedValueOnce(broken());
+ await searchGovIfRouted(search, "s1", logger);
+ await searchGovIfRouted(search, "s1", logger);
+ }
+ const asked = fetchMock.mock.calls.length;
+
+ await expect(searchGovIfRouted(search, "s1", logger)).resolves.toBeNull();
+
+ expect(fetchMock).toHaveBeenCalledTimes(asked);
+ expect(outcomes().at(-1)).toBe("paused");
+ expect(shared.rootLogger.warn).toHaveBeenCalledWith(
+ "Gov routing paused after repeated search platform failures",
+ expect.objectContaining({ call: "webSearch" }),
+ );
+ },
+ );
+
+ it("starts the failure count over after the platform answers", async () => {
+ for (let attempt = 0; attempt < 4; attempt++) {
+ fetchMock.mockRejectedValueOnce(new Error("timeout"));
+ await searchGovIfRouted(search, "s1", logger);
+ }
+ fetchMock.mockResolvedValueOnce(vertical({ search_category_routed: [] }));
+ await searchGovIfRouted(search, "s1", logger);
+ fetchMock.mockRejectedValueOnce(new Error("timeout"));
+ await searchGovIfRouted(search, "s1", logger);
+
+ expect(outcomes()).not.toContain("paused");
+ expect(fetchMock).toHaveBeenCalledTimes(6);
+ });
+
+ /**
+ * The pause is shared by every team on the process, so a caller must not be
+ * able to trip it: a refused query says nothing about the platform's health.
+ */
+ it("does not pause on queries the search platform refuses", async () => {
+ for (let attempt = 0; attempt < 5; attempt++) {
+ fetchMock.mockResolvedValueOnce({
+ ok: false,
+ status: 400,
+ body: { cancel: async () => {} },
+ } as any);
+ await searchGovIfRouted(search, "s1", logger);
+ }
+ fetchMock.mockResolvedValueOnce(vertical({ search_category_routed: [] }));
+
+ await searchGovIfRouted(search, "s1", logger);
+
+ expect(fetchMock).toHaveBeenCalledTimes(6);
+ expect(outcomes()).toEqual([
+ ...Array(5).fill("decision_failed"),
+ "not_routed",
+ ]);
+ });
+
+ /**
+ * A routed search may take at most half the time left in the request, so the
+ * regular web search keeps time to run when the route fails. A timeout set by
+ * that deadline comes from the caller's own short `timeout`, so it must not
+ * count toward the pause; a timeout at the routed search's own limit does.
+ */
+ it.each([
+ ["the request's own deadline", 500, "not_routed"],
+ ["the routed search's own limit", 60_000, "paused"],
+ ])(
+ "on five routed searches in a row timed out by %s (timeout %i), the next search is %s",
+ async (_case, timeout, next) => {
+ const timedOut = () =>
+ new DOMException(
+ "The operation was aborted due to timeout",
+ "TimeoutError",
+ );
+ for (let attempt = 0; attempt < 5; attempt++) {
+ fetchMock
+ .mockResolvedValueOnce(routedToGov())
+ .mockRejectedValueOnce(timedOut());
+ await searchGovIfRouted({ ...search, timeout }, "s1", logger);
+ }
+ fetchMock.mockResolvedValueOnce(vertical({ search_category_routed: [] }));
+
+ await searchGovIfRouted({ ...search, timeout }, "s1", logger);
+
+ expect(outcomes()).toEqual([...Array(5).fill("routed_failed"), next]);
+ },
+ );
+
+ it("starts no routed search for a decision that was in flight when routing paused", async () => {
+ let answerDecision!: (response: unknown) => void;
+ fetchMock.mockImplementationOnce(
+ () =>
+ new Promise(resolve => {
+ answerDecision = resolve;
+ }),
+ );
+ const inFlight = searchGovIfRouted(search, "s0", logger);
+ for (let attempt = 0; attempt < 5; attempt++) {
+ fetchMock.mockRejectedValueOnce(new Error("connect ECONNREFUSED"));
+ await searchGovIfRouted(search, "s1", logger);
+ }
+
+ answerDecision(routedToGov());
+
+ await expect(inFlight).resolves.toBeNull();
+ expect(fetchMock).toHaveBeenCalledTimes(6);
+ expect(outcomes()).toEqual([...Array(5).fill("decision_failed"), "paused"]);
+ });
+});
diff --git a/apps/api/src/search/gov.ts b/apps/api/src/search/gov.ts
index 2456aa835..43e0d6dd1 100644
--- a/apps/api/src/search/gov.ts
+++ b/apps/api/src/search/gov.ts
@@ -1,13 +1,65 @@
import type { Logger } from "winston";
import type { WebSearchResult } from "../lib/entities";
+import { config } from "../config";
import {
fetchGovUpstream,
+ fetchQueryFeatures,
+ fetchWebSearchUpstream,
usOnlySkippedWarning,
usOnlyUpstreamRefusal,
UsOnlyUpstreamRefusalError,
} from "../lib/research-upstream";
+import { logger as rootLogger } from "../lib/logger";
import { hasCategory, type CategoryOption } from "../lib/search-query-builder";
+/**
+ * The longest a routed search waits for the search platform before falling
+ * back to the regular web search, which starts only then.
+ */
+const ROUTED_SEARCH_TIMEOUT_MS = 4000;
+
+/**
+ * The largest page a search may ask for and still be routed. A routed answer
+ * may hold no more than this many results, so a larger page stays on the
+ * regular web search.
+ */
+export const MAX_ROUTED_LIMIT = 10;
+
+/**
+ * After this many failures in a row of one search platform call, this process
+ * stops routing for PAUSE_MS. An outage then delays a handful of searches per
+ * process instead of every routed search until routing is switched off.
+ */
+const PAUSE_AFTER_FAILURES = 5;
+const PAUSE_MS = 60_000;
+
+/**
+ * Failures in a row of each search platform call on this process. Only a call's
+ * own answer resets its count: a decision that routes nothing says nothing
+ * about the web search.
+ */
+const failuresInARow = { decision: 0, webSearch: 0 };
+let pausedUntil = 0;
+
+/**
+ * What one search platform call says about the platform's health. A refusal
+ * of the query (a 4xx) or a call cut short by the request's own deadline says
+ * nothing either way, so no caller can pause routing for every team on the
+ * process.
+ */
+type PlatformHealth = "answered" | "failed" | "unknown";
+
+type GovSearchOptions = Parameters<typeof searchGovCategory>[0];
+
+/** What happened to one plain web search's routing; logged once per search. */
+type RoutingOutcome =
+ | "paused"
+ | "decision_failed"
+ | "not_routed"
+ | "routed_failed"
+ | "routed_empty"
+ | "routed_served";
+
export function wantsGovCategory(categories?: CategoryOption[]): boolean {
return hasCategory(categories, "gov");
}
@@ -56,27 +108,265 @@ export async function searchGovCategory(
: undefined;
if (skipped) options.warnings?.push(skipped);
- const body: any = await upstream.json();
- const results: any[] = Array.isArray(body?.data?.web) ? body.data.web : [];
- return results
- .slice(0, options.limit)
- .map((result, index) => ({
- url: typeof result?.url === "string" ? result.url : "",
- title:
- typeof result?.title === "string" && result.title.trim().length > 0
- ? result.title
- : typeof result?.url === "string"
- ? result.url
- : "",
- description:
- typeof result?.description === "string" ? result.description : "",
- position: index + 1,
- category: "gov",
- }))
- .filter(result => result.url.length > 0);
+ return platformResults(await upstream.json(), options.limit).map(
+ result => ({ ...result, category: "gov" }),
+ );
} catch (error) {
if (error instanceof UsOnlyUpstreamRefusalError) throw error;
logger.warn("Gov category upstream error", { error });
return [];
}
}
+
+/**
+ * The search platform's results for a plain web search that it classifies as
+ * government, or null when the search stays on the regular web search: routing
+ * is off or paused, the decision failed or did not route the query, or the
+ * routed search failed or found nothing. An answer with fewer results than
+ * asked is served as it is, as a short page of regular web results is.
+ *
+ * Routing spends the request's own `timeout`, and leaves at least half of it to
+ * the regular web search when the route fails.
+ *
+ * Every search that reaches the routing decision logs one `search_gov_routing`
+ * line with its outcome.
+ */
+export async function searchGovIfRouted(
+ options: GovSearchOptions,
+ searchId: string,
+ logger: Logger,
+): Promise<WebSearchResult[] | null> {
+ if (!config.GOV_ROUTING_ENABLED) return null;
+ const deadline = Date.now() + options.timeout;
+
+ const record = (
+ outcome: RoutingOutcome,
+ measured: { decisionMs?: number; webSearchMs?: number; results?: number },
+ ) =>
+ // Root logger: the request logger carries the query, and this line records shape only.
+ rootLogger.info("Search gov routing", {
+ event: "search_gov_routing",
+ searchId,
+ teamId: options.teamId,
+ outcome,
+ limit: options.limit,
+ ...measured,
+ });
+
+ if (Date.now() < pausedUntil) {
+ record("paused", {});
+ return null;
+ }
+
+ const decisionStarted = Date.now();
+ const decision = await searchCategoriesRouted(options, deadline, logger);
+ const decisionMs = Date.now() - decisionStarted;
+ countCall("decision", decision.health);
+ if (decision.categories === null) {
+ record("decision_failed", { decisionMs });
+ return null;
+ }
+ if (!decision.categories.includes("GOVERNMENT")) {
+ record("not_routed", { decisionMs });
+ return null;
+ }
+ // Routing may have paused while this decision was in flight.
+ if (Date.now() < pausedUntil) {
+ record("paused", { decisionMs });
+ return null;
+ }
+
+ const webSearchStarted = Date.now();
+ const routed = await searchRoutedWeb(options, deadline, logger);
+ const webSearchMs = Date.now() - webSearchStarted;
+ countCall("webSearch", routed.health);
+ if (routed.results === null) {
+ record("routed_failed", { decisionMs, webSearchMs });
+ return null;
+ }
+ if (routed.results.length === 0) {
+ record("routed_empty", { decisionMs, webSearchMs, results: 0 });
+ return null;
+ }
+ record("routed_served", {
+ decisionMs,
+ webSearchMs,
+ results: routed.results.length,
+ });
+ return routed.results;
+}
+
+/**
+ * Counts what a search platform call said about the platform's health, and
+ * pauses routing on this process once one call has failed
+ * PAUSE_AFTER_FAILURES times in a row.
+ */
+function countCall(
+ call: keyof typeof failuresInARow,
+ health: PlatformHealth,
+): void {
+ if (health === "unknown") return;
+ if (health === "answered") {
+ failuresInARow[call] = 0;
+ return;
+ }
+ failuresInARow[call] += 1;
+ if (failuresInARow[call] < PAUSE_AFTER_FAILURES) return;
+ failuresInARow.decision = 0;
+ failuresInARow.webSearch = 0;
+ pausedUntil = Date.now() + PAUSE_MS;
+ rootLogger.warn(
+ "Gov routing paused after repeated search platform failures",
+ {
+ call,
+ failuresInARow: PAUSE_AFTER_FAILURES,
+ pauseMs: PAUSE_MS,
+ },
+ );
+}
+
+/**
+ * How long a routing call may take: its own limit, or half the time left
+ * before the request's deadline when that is shorter, so a failed route
+ * leaves the regular web search at least as long as the route took.
+ * `deadlineBound` is true when the deadline, not the limit, set the time.
+ */
+function routingTimeout(limitMs: number, deadline: number) {
+ const halfLeft = Math.max(0, Math.floor((deadline - Date.now()) / 2));
+ return halfLeft < limitMs
+ ? { timeoutMs: halfLeft, deadlineBound: true }
+ : { timeoutMs: limitMs, deadlineBound: false };
+}
+
+/**
+ * The search platform's web search for a routed query, with null results when
+ * the request failed, timed out, or was answered with a status other than 2xx
+ * or a body without a `data.web` list. It asks for twice the page, as the
+ * regular web search does, so results that safe search or threat protection
+ * remove still leave a full page. Its results are served as web results, so
+ * they carry no gov category.
+ */
+async function searchRoutedWeb(
+ options: GovSearchOptions,
+ deadline: number,
+ logger: Logger,
+): Promise<{ results: WebSearchResult[] | null; health: PlatformHealth }> {
+ const { timeoutMs, deadlineBound } = routingTimeout(
+ ROUTED_SEARCH_TIMEOUT_MS,
+ deadline,
+ );
+ const k = options.limit * 2;
+ try {
+ const upstream = await fetchWebSearchUpstream({
+ query: options.query,
+ k,
+ headers: {
+ "firecrawl-team-id": options.teamId,
+ "firecrawl-zero-data-retention": String(options.zeroDataRetention),
+ },
+ timeoutMs,
+ });
+ if (upstream === null) return { results: null, health: "failed" };
+ if (!upstream.ok) {
+ await upstream.body?.cancel().catch(() => {});
+ logger.warn("Routed web search upstream failed", {
+ status: upstream.status,
+ });
+ return {
+ results: null,
+ health:
+ upstream.status >= 400 && upstream.status < 500
+ ? "unknown"
+ : "failed",
+ };
+ }
+ const body: any = await upstream.json();
+ if (!Array.isArray(body?.data?.web)) {
+ logger.warn("Routed web search answered without data.web");
+ return { results: null, health: "failed" };
+ }
+ return { results: platformResults(body, k), health: "answered" };
+ } catch (error) {
+ logger.warn("Routed web search upstream error", { error });
+ return {
+ results: null,
+ health:
+ deadlineBound && error?.name === "TimeoutError" ? "unknown" : "failed",
+ };
+ }
+}
+
+/** The `data.web` results of a search platform answer, renumbered from 1. */
+function platformResults(body: any, limit: number): WebSearchResult[] {
+ const results: any[] = Array.isArray(body?.data?.web) ? body.data.web : [];
+ return results
+ .slice(0, limit)
+ .map((result, index) => ({
+ url: typeof result?.url === "string" ? result.url : "",
+ title:
+ typeof result?.title === "string" && result.title.trim().length > 0
+ ? result.title
+ : typeof result?.url === "string"
+ ? result.url
+ : "",
+ description:
+ typeof result?.description === "string" ? result.description : "",
+ position: index + 1,
+ }))
+ .filter(result => result.url.length > 0);
+}
+
+/**
+ * The categories the search platform routes the query to, null when the call
+ * failed, timed out, or answered without a list of category names.
+ */
+async function searchCategoriesRouted(
+ options: GovSearchOptions,
+ deadline: number,
+ logger: Logger,
+): Promise<{ categories: string[] | null; health: PlatformHealth }> {
+ const { timeoutMs, deadlineBound } = routingTimeout(
+ config.GOV_ROUTING_DECISION_TIMEOUT_MS,
+ deadline,
+ );
+ try {
+ const upstream = await fetchQueryFeatures({
+ query: options.query,
+ headers: {
+ "firecrawl-zero-data-retention": String(options.zeroDataRetention),
+ },
+ timeoutMs,
+ });
+ if (upstream === null) return { categories: null, health: "failed" };
+ if (!upstream.ok) {
+ await upstream.body?.cancel().catch(() => {});
+ logger.warn("Routing decision upstream failed", {
+ status: upstream.status,
+ });
+ return {
+ categories: null,
+ health:
+ upstream.status >= 400 && upstream.status < 500
+ ? "unknown"
+ : "failed",
+ };
+ }
+ const body: any = await upstream.json();
+ const routed = body?.results?.[0]?.vertical?.search_category_routed;
+ if (
+ !Array.isArray(routed) ||
+ !routed.every(category => typeof category === "string")
+ ) {
+ logger.warn("Routing decision answered without search_category_routed");
+ return { categories: null, health: "failed" };
+ }
+ return { categories: routed, health: "answered" };
+ } catch (error) {
+ logger.warn("Routing decision upstream error", { error });
+ return {
+ categories: null,
+ health:
+ deadlineBound && error?.name === "TimeoutError" ? "unknown" : "failed",
+ };
+ }
+}
diff --git a/apps/api/src/search/operators.ts b/apps/api/src/search/operators.ts
index 7ff2930b6..ac2cfd375 100644
--- a/apps/api/src/search/operators.ts
+++ b/apps/api/src/search/operators.ts
@@ -54,6 +54,19 @@ function hasUnsupportedOr(query: string): boolean {
return ors > siteOrs;
}
+/**
+ * True when the query uses search syntax: a quoted phrase, a `name:value`
+ * token such as `site:` or `filetype:`, or an OR or `|` between terms. A
+ * routed search sends the query text alone, so it carries none of these.
+ */
+export function usesSearchSyntax(query: string): boolean {
+ if (query.includes('"')) return true;
+ return (
+ /(?:^|[\s(-])[a-z]+:\S/i.test(query) ||
+ /(?:^|[\s(])(?:OR|\|)(?=[\s)(]|$)/.test(query)
+ );
+}
+
export function parseQueryOperators(query: string): QueryOperators {
const operators = emptyOperators();
for (const match of query.matchAll(OPERATOR_TOKEN)) {