Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): return Government Index labels as gov results on /v2/search/gov (#5354)
Erik Engervall16fa127831e7
diff --git a/apps/api/openapi.json b/apps/api/openapi.jsonindex 238d10c99..8670db680 100644--- a/apps/api/openapi.json+++ b/apps/api/openapi.json@@ -6389,12 +6389,19 @@"name": "cites","in": "query","required": false,- "description": "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`.",+ "description": "Keep only pages that cite a law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`. Pass up to 32 to keep pages citing any of them. Repeat the parameter to pass more than one.","schema": {- "type": "string",- "minLength": 1,- "example": "42 U.S.C. § 1983"- }+ "type": "array",+ "items": {+ "type": "string",+ "minLength": 1+ },+ "minItems": 1,+ "maxItems": 32,+ "example": ["42 U.S.C. § 1983"]+ },+ "style": "form",+ "explode": true},{"name": "domain",@@ -6406,11 +6413,28 @@"minLength": 1,"example": "irs.gov"}+ },+ {+ "name": "sources",+ "in": "query",+ "required": false,+ "description": "Result groups to return. `web` returns plain search results. `gov` returns the same results with the index's labels: `jurisdiction`, `court`, `courtType`, `cites`, `citedAs`, `lang` and `updatedAt`. Repeat the parameter to return both.",+ "schema": {+ "type": "array",+ "items": {+ "type": "string",+ "enum": ["web", "gov"]+ },+ "minItems": 1,+ "default": ["web"]+ },+ "style": "form",+ "explode": true}],"responses": {"200": {- "description": "Ranked Government Index results in the `web` group.",+ "description": "Ranked Government Index results in each group `sources` names, `web` by default.","content": {"application/json": {"schema": {@@ -6509,9 +6533,22 @@"description": "Keep only pages from one type of court."},"cites": {- "type": "string",- "minLength": 1,- "description": "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`.",+ "oneOf": [+ {+ "type": "string",+ "minLength": 1+ },+ {+ "type": "array",+ "items": {+ "type": "string",+ "minLength": 1+ },+ "minItems": 1,+ "maxItems": 32+ }+ ],+ "description": "Keep only pages that cite a law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`. Pass up to 32 to keep pages citing any of them.","example": "42 U.S.C. § 1983"},"domain": {@@ -6519,6 +6556,24 @@"minLength": 1,"description": "Keep only pages from one site, such as `irs.gov`.","example": "irs.gov"+ },+ "sources": {+ "oneOf": [+ {+ "type": "string",+ "enum": ["web", "gov"]+ },+ {+ "type": "array",+ "items": {+ "type": "string",+ "enum": ["web", "gov"]+ },+ "minItems": 1+ }+ ],+ "default": ["web"],+ "description": "Result groups to return, as one name or a list. `web` returns plain search results. `gov` returns the same results with the index's labels: `jurisdiction`, `court`, `courtType`, `cites`, `citedAs`, `lang` and `updatedAt`."}}},@@ -6531,7 +6586,7 @@},"responses": {"200": {- "description": "Ranked Government Index results in the `web` group.",+ "description": "Ranked Government Index results in each group `sources` names, `web` by default.","content": {"application/json": {"schema": {@@ -12169,6 +12224,65 @@}}}+ },+ "gov": {+ "type": "array",+ "description": "Present when `sources` includes `gov`. The same results as `web`, in the same order, with the index's labels.",+ "items": {+ "type": "object",+ "properties": {+ "url": {+ "type": "string",+ "format": "uri"+ },+ "title": {+ "type": "string"+ },+ "description": {+ "type": "string",+ "description": "Matched snippet from the source document."+ },+ "position": {+ "type": "integer",+ "description": "1-based rank in the result list."+ },+ "jurisdiction": {+ "type": "string",+ "description": "Jurisdiction label: `us` for federal, or `us-` plus a two-letter state or territory code. Absent when the index has no label."+ },+ "court": {+ "type": "string",+ "description": "CourtListener court id, such as `scotus` or `ca9`. Absent when the index has no label."+ },+ "courtType": {+ "type": "string",+ "description": "Type of court, one of the `courtType` filter values. Absent when the index has no label."+ },+ "cites": {+ "type": "array",+ "items": {+ "type": "string"+ },+ "description": "Citation ids of the laws and cases the page cites. Absent when none are known."+ },+ "citedAs": {+ "type": "array",+ "items": {+ "type": "string"+ },+ "description": "Citation ids other pages cite this page by. Pass one as `cites` to find the pages citing it. Absent when none are known."+ },+ "lang": {+ "type": "string",+ "description": "Language code of the page, such as `en`. Absent when unknown."+ },+ "updatedAt": {+ "type": "integer",+ "description": "When the page was crawled into the index, in Unix milliseconds. Omitted for results that were never crawled into the index."+ }+ },+ "required": ["url", "title", "description", "position"]+ }}}}diff --git a/apps/api/src/__tests__/snips/v2/gov.test.ts b/apps/api/src/__tests__/snips/v2/gov.test.tsindex f6e86479e..7e8d87162 100644--- a/apps/api/src/__tests__/snips/v2/gov.test.ts+++ b/apps/api/src/__tests__/snips/v2/gov.test.ts@@ -1,6 +1,6 @@import { config } from "../../../config";import { describeIf } from "../lib";-import { idmux, researchRaw } from "./lib";+import { idmux, researchRaw, scrapeTimeout } from "./lib";const HAS_GOV = !!config.SEARCH_PLATFORM_URL;const PATH = "/v2/search/gov";@@ -30,4 +30,81 @@ describeIf(HAS_GOV)("Government Index Search API", () => {expect(typeof result.position).toBe("number");}}, 120000);++ it(+ "returns gov results with index labels and keeps web results plain",+ async () => {+ const identity = await idmux({ name: "gov/gov-group", credits: 100 });++ const res = await researchRaw(+ PATH,+ {+ query: "qualified immunity excessive force",+ k: 5,+ jurisdiction: "us",+ sources: ["web", "gov"],+ },+ identity,+ );++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ const { web, gov } = res.body.data;+ expect(Array.isArray(gov)).toBe(true);+ expect(gov.length).toBeGreaterThan(0);+ expect(gov.map((r: any) => r.url)).toEqual(web.map((r: any) => r.url));+ for (const result of web) {+ for (const key of [+ "jurisdiction",+ "court",+ "courtType",+ "cites",+ "citedAs",+ "updatedAt",+ "lang",+ ]) {+ expect(result[key]).toBeUndefined();+ }+ }+ for (const result of gov) {+ expect(typeof result.url).toBe("string");+ expect(typeof result.position).toBe("number");+ for (const key of ["court_type", "cited_as", "updated_at"]) {+ expect(result[key]).toBeUndefined();+ }+ if (result.updatedAt !== undefined) {+ expect(typeof result.updatedAt).toBe("number");+ }+ if (result.lang !== undefined) {+ expect(typeof result.lang).toBe("string");+ }+ if (result.jurisdiction !== undefined) {+ expect(result.jurisdiction).toBe("us");+ }+ for (const key of ["cites", "citedAs"]) {+ if (result[key] !== undefined) {+ expect(Array.isArray(result[key])).toBe(true);+ }+ }+ }+ },+ scrapeTimeout,+ );++ it(+ "refuses a source other than web or gov",+ async () => {+ const identity = await idmux({ name: "gov/bad-source", credits: 100 });++ const res = await researchRaw(+ PATH,+ { query: "zoning", sources: ["news"] },+ identity,+ );++ expect(res.statusCode).toBe(400);+ expect(res.body.success).toBe(false);+ },+ scrapeTimeout,+ );});diff --git a/apps/api/src/controllers/v2/__tests__/gov-search.test.ts b/apps/api/src/controllers/v2/__tests__/gov-search.test.tsindex 6b80beaf4..c207d1a41 100644--- a/apps/api/src/controllers/v2/__tests__/gov-search.test.ts+++ b/apps/api/src/controllers/v2/__tests__/gov-search.test.ts@@ -188,6 +188,18 @@ describe("/v2/search/gov", () => {["GET", { query: "rules", k: "101" }],["GET", { query: "rules", magic: "true" }],["POST", { query: "rules", magic: "true" }],+ ["GET", { query: "rules", sources: "news" }],+ ["POST", { query: "rules", sources: [] }],+ ["POST", { query: "rules", sources: ["web", "news"] }],+ ["POST", { query: "rules", sources: ["law"] }],+ ["POST", { query: "rules", cites: [] }],+ [+ "POST",+ {+ query: "rules",+ cites: Array.from({ length: 33 }, (_, i) => "us:usc:42:" + i),+ },+ ],] as const)("rejects invalid %s input %j before the upstream call",async (method, input) => {@@ -203,6 +215,96 @@ describe("/v2/search/gov", () => {},);+ it.each([+ ["GET", ["gov", "gov"]],+ ["GET", "gov"],+ ["POST", ["gov"]],+ ] as const)(+ "forwards a %s request for gov results and counts them",+ async (method, sources) => {+ const labels = {+ jurisdiction: "us",+ court: "scotus",+ cites: ["us:usc:21:343"],+ lang: "en",+ };+ const gov = [+ {+ ...WEB_RESULT,+ ...labels,+ court_type: "federal-supreme",+ cited_as: ["us:us-reports:555:223"],+ updated_at: 1700000000000,+ },+ ];+ mocks.fetchGovUpstream.mockResolvedValue({+ ok: true,+ status: 200,+ headers: new Headers(),+ text: async () => JSON.stringify({ success: true, data: { gov } }),+ });+ const res = makeRes();+ await handler(method === "GET" ? "get" : "post")(+ makeReq(method, { query: "food labeling rules", sources }),+ res,+ );+ await flush();++ expect(mocks.fetchGovUpstream).toHaveBeenCalledWith(+ expect.objectContaining({ sources: ["gov"] }),+ );+ expect(res.json).toHaveBeenCalledWith({+ success: true,+ data: {+ gov: [+ {+ ...WEB_RESULT,+ ...labels,+ courtType: "federal-supreme",+ citedAs: ["us:us-reports:555:223"],+ updatedAt: 1700000000000,+ },+ ],+ },+ });+ expect(mocks.logResearchEndpoint).toHaveBeenCalledWith(+ expect.objectContaining({ num_results: 1, credits_cost: 0 }),+ );+ },+ );++ it.each([+ ["GET", "us:usc:42:1983", "us:usc:42:1983"],+ [+ "GET",+ ["us:usc:42:1983", "us:us-reports:5:137"],+ ["us:usc:42:1983", "us:us-reports:5:137"],+ ],+ ["POST", [" us:usc:42:1983 ", "us:usc:42:1983"], "us:usc:42:1983"],+ ] as const)(+ "forwards a %s cites filter %j as %j",+ async (method, cites, expected) => {+ const res = makeRes();+ await handler(method === "GET" ? "get" : "post")(+ makeReq(method, { query: "civil rights", cites }),+ res,+ );+ await flush();++ expect(mocks.fetchGovUpstream).toHaveBeenCalledWith(+ expect.objectContaining({ cites: expected }),+ );+ },+ );++ it("leaves sources out of the upstream call unless sent", async () => {+ const res = makeRes();+ await handler("post")(makeReq("POST", { query: "zoning variance" }), res);+ await flush();++ expect(mocks.fetchGovUpstream.mock.calls[0][0].sources).toBeUndefined();+ });+it("maps an upstream timeout to 504", async () => {mocks.fetchGovUpstream.mockRejectedValue(new DOMException("timed out", "TimeoutError"),diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex d60110f8d..81d8af909 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -161,6 +161,12 @@ const DEVELOPER_SEARCH_QUERY_KEYS = ["skills",];+const govSource = z.enum(["web", "gov"]);++const govCite = z.string().trim().min(1);++const govCites = z.array(govCite).min(1).max(32);+const govSearchSchema = z.strictObject({query: z.string().min(1),k: kSchema(100),@@ -189,8 +195,31 @@ const govSearchSchema = z.strictObject({"state-attorney-general",]).optional(),- cites: z.string().trim().min(1).optional(),+ cites: z+ .unknown()+ .transform((value, ctx) => {+ const parsed = (Array.isArray(value) ? govCites : govCite).safeParse(+ value,+ );+ if (!parsed.success) {+ ctx.issues.push(+ ...parsed.error.issues.map(+ issue => ({ ...issue, input: value }) as z.core.$ZodRawIssue,+ ),+ );+ return z.NEVER;+ }+ const cites = [...new Set([parsed.data].flat())];+ return cites.length === 1 ? cites[0] : cites;+ })+ .optional(),domain: z.string().trim().min(1).optional(),+ sources: z+ .union([govSource, z.array(govSource).min(1)])+ .optional()+ .transform(value =>+ value === undefined ? undefined : [...new Set([value].flat())],+ ),...commonQuery,});@@ -267,9 +296,30 @@ function addLegacySnakeCaseAliases<T>(value: T): T {return object as T;}+const GOV_RESULT_KEYS: Record<string, string> = {+ court_type: "courtType",+ cited_as: "citedAs",+ updated_at: "updatedAt",+};++function camelCaseGovResults(body: any) {+ const gov = body?.data?.gov;+ if (!Array.isArray(gov)) return body;+ body.data.gov = gov.map(item =>+ Object.fromEntries(+ Object.entries(item).map(([key, value]) => [+ GOV_RESULT_KEYS[key] ?? key,+ value,+ ]),+ ),+ );+ return body;+}+function resultCount(body: any): number {if (Array.isArray(body?.results)) return body.results.length;- return Array.isArray(body?.data?.web) ? body.data.web.length : 0;+ const group = body?.data?.web ?? body?.data?.gov;+ return Array.isArray(group) ? group.length : 0;}function creditsFor(@@ -508,10 +558,13 @@ function createResearchController(if (responseBody === null || typeof responseBody === "string") {return res.status(statusCode).send(responseBody ?? "");}- const response =- options.legacy && upstream.ok+ const response = !upstream.ok+ ? responseBody+ : options.legacy? addLegacySnakeCaseAliases(responseBody)- : responseBody;+ : endpoint.kind === "gov_search"+ ? camelCaseGovResults(responseBody)+ : responseBody;const usOnlyWarning =upstream.ok && isUsOnly(authedReq.acuc?.flags)? usOnlySkippedWarning(upstream.headers)@@ -712,6 +765,7 @@ export function createGovRouter() {courtType: params.courtType,cites: params.cites,domain: params.domain,+ sources: params.sources,headers,timeoutMs: GOV_SEARCH_TIMEOUT_MS,}),diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.tsindex af095bdb2..ddb0b7c8c 100644--- a/apps/api/src/lib/research-upstream.ts+++ b/apps/api/src/lib/research-upstream.ts@@ -121,8 +121,9 @@ export async function fetchGovUpstream(options: {jurisdiction?: string;court?: string;courtType?: string;- cites?: string;+ cites?: string | string[];domain?: string;+ sources?: ("web" | "gov")[];headers: Record<string, string>;timeoutMs: number;}) {@@ -136,6 +137,7 @@ export async function fetchGovUpstream(options: {court_type: options.courtType,cites: options.cites,domain: options.domain,+ sources: options.sources,},options.headers,options.timeoutMs,