Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): return Government Index labels as gov results on /v2/search/gov (#5354)

Erik Engervall16fa127831e7
diff --git a/apps/api/openapi.json b/apps/api/openapi.json
index 238d10c99..8670db680 100644
--- a/apps/api/openapi.json
+++ b/apps/api/openapi.json
@@ -6389,12 +6389,19 @@
"name": "cites",
"in": "query",
"required": false,
- "description": "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`.",
+ "description": "Keep only pages that cite a law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`. Pass up to 32 to keep pages citing any of them. Repeat the parameter to pass more than one.",
"schema": {
- "type": "string",
- "minLength": 1,
- "example": "42 U.S.C. § 1983"
- }
+ "type": "array",
+ "items": {
+ "type": "string",
+ "minLength": 1
+ },
+ "minItems": 1,
+ "maxItems": 32,
+ "example": ["42 U.S.C. § 1983"]
+ },
+ "style": "form",
+ "explode": true
},
{
"name": "domain",
@@ -6406,11 +6413,28 @@
"minLength": 1,
"example": "irs.gov"
}
+ },
+ {
+ "name": "sources",
+ "in": "query",
+ "required": false,
+ "description": "Result groups to return. `web` returns plain search results. `gov` returns the same results with the index's labels: `jurisdiction`, `court`, `courtType`, `cites`, `citedAs`, `lang` and `updatedAt`. Repeat the parameter to return both.",
+ "schema": {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "enum": ["web", "gov"]
+ },
+ "minItems": 1,
+ "default": ["web"]
+ },
+ "style": "form",
+ "explode": true
}
],
"responses": {
"200": {
- "description": "Ranked Government Index results in the `web` group.",
+ "description": "Ranked Government Index results in each group `sources` names, `web` by default.",
"content": {
"application/json": {
"schema": {
@@ -6509,9 +6533,22 @@
"description": "Keep only pages from one type of court."
},
"cites": {
- "type": "string",
- "minLength": 1,
- "description": "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`.",
+ "oneOf": [
+ {
+ "type": "string",
+ "minLength": 1
+ },
+ {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "minLength": 1
+ },
+ "minItems": 1,
+ "maxItems": 32
+ }
+ ],
+ "description": "Keep only pages that cite a law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`. Pass up to 32 to keep pages citing any of them.",
"example": "42 U.S.C. § 1983"
},
"domain": {
@@ -6519,6 +6556,24 @@
"minLength": 1,
"description": "Keep only pages from one site, such as `irs.gov`.",
"example": "irs.gov"
+ },
+ "sources": {
+ "oneOf": [
+ {
+ "type": "string",
+ "enum": ["web", "gov"]
+ },
+ {
+ "type": "array",
+ "items": {
+ "type": "string",
+ "enum": ["web", "gov"]
+ },
+ "minItems": 1
+ }
+ ],
+ "default": ["web"],
+ "description": "Result groups to return, as one name or a list. `web` returns plain search results. `gov` returns the same results with the index's labels: `jurisdiction`, `court`, `courtType`, `cites`, `citedAs`, `lang` and `updatedAt`."
}
}
},
@@ -6531,7 +6586,7 @@
},
"responses": {
"200": {
- "description": "Ranked Government Index results in the `web` group.",
+ "description": "Ranked Government Index results in each group `sources` names, `web` by default.",
"content": {
"application/json": {
"schema": {
@@ -12169,6 +12224,65 @@
}
}
}
+ },
+ "gov": {
+ "type": "array",
+ "description": "Present when `sources` includes `gov`. The same results as `web`, in the same order, with the index's labels.",
+ "items": {
+ "type": "object",
+ "properties": {
+ "url": {
+ "type": "string",
+ "format": "uri"
+ },
+ "title": {
+ "type": "string"
+ },
+ "description": {
+ "type": "string",
+ "description": "Matched snippet from the source document."
+ },
+ "position": {
+ "type": "integer",
+ "description": "1-based rank in the result list."
+ },
+ "jurisdiction": {
+ "type": "string",
+ "description": "Jurisdiction label: `us` for federal, or `us-` plus a two-letter state or territory code. Absent when the index has no label."
+ },
+ "court": {
+ "type": "string",
+ "description": "CourtListener court id, such as `scotus` or `ca9`. Absent when the index has no label."
+ },
+ "courtType": {
+ "type": "string",
+ "description": "Type of court, one of the `courtType` filter values. Absent when the index has no label."
+ },
+ "cites": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "Citation ids of the laws and cases the page cites. Absent when none are known."
+ },
+ "citedAs": {
+ "type": "array",
+ "items": {
+ "type": "string"
+ },
+ "description": "Citation ids other pages cite this page by. Pass one as `cites` to find the pages citing it. Absent when none are known."
+ },
+ "lang": {
+ "type": "string",
+ "description": "Language code of the page, such as `en`. Absent when unknown."
+ },
+ "updatedAt": {
+ "type": "integer",
+ "description": "When the page was crawled into the index, in Unix milliseconds. Omitted for results that were never crawled into the index."
+ }
+ },
+ "required": ["url", "title", "description", "position"]
+ }
}
}
}
diff --git a/apps/api/src/__tests__/snips/v2/gov.test.ts b/apps/api/src/__tests__/snips/v2/gov.test.ts
index f6e86479e..7e8d87162 100644
--- a/apps/api/src/__tests__/snips/v2/gov.test.ts
+++ b/apps/api/src/__tests__/snips/v2/gov.test.ts
@@ -1,6 +1,6 @@
import { config } from "../../../config";
import { describeIf } from "../lib";
-import { idmux, researchRaw } from "./lib";
+import { idmux, researchRaw, scrapeTimeout } from "./lib";
const HAS_GOV = !!config.SEARCH_PLATFORM_URL;
const PATH = "/v2/search/gov";
@@ -30,4 +30,81 @@ describeIf(HAS_GOV)("Government Index Search API", () => {
expect(typeof result.position).toBe("number");
}
}, 120000);
+
+ it(
+ "returns gov results with index labels and keeps web results plain",
+ async () => {
+ const identity = await idmux({ name: "gov/gov-group", credits: 100 });
+
+ const res = await researchRaw(
+ PATH,
+ {
+ query: "qualified immunity excessive force",
+ k: 5,
+ jurisdiction: "us",
+ sources: ["web", "gov"],
+ },
+ identity,
+ );
+
+ expect(res.statusCode).toBe(200);
+ expect(res.body.success).toBe(true);
+ const { web, gov } = res.body.data;
+ expect(Array.isArray(gov)).toBe(true);
+ expect(gov.length).toBeGreaterThan(0);
+ expect(gov.map((r: any) => r.url)).toEqual(web.map((r: any) => r.url));
+ for (const result of web) {
+ for (const key of [
+ "jurisdiction",
+ "court",
+ "courtType",
+ "cites",
+ "citedAs",
+ "updatedAt",
+ "lang",
+ ]) {
+ expect(result[key]).toBeUndefined();
+ }
+ }
+ for (const result of gov) {
+ expect(typeof result.url).toBe("string");
+ expect(typeof result.position).toBe("number");
+ for (const key of ["court_type", "cited_as", "updated_at"]) {
+ expect(result[key]).toBeUndefined();
+ }
+ if (result.updatedAt !== undefined) {
+ expect(typeof result.updatedAt).toBe("number");
+ }
+ if (result.lang !== undefined) {
+ expect(typeof result.lang).toBe("string");
+ }
+ if (result.jurisdiction !== undefined) {
+ expect(result.jurisdiction).toBe("us");
+ }
+ for (const key of ["cites", "citedAs"]) {
+ if (result[key] !== undefined) {
+ expect(Array.isArray(result[key])).toBe(true);
+ }
+ }
+ }
+ },
+ scrapeTimeout,
+ );
+
+ it(
+ "refuses a source other than web or gov",
+ async () => {
+ const identity = await idmux({ name: "gov/bad-source", credits: 100 });
+
+ const res = await researchRaw(
+ PATH,
+ { query: "zoning", sources: ["news"] },
+ identity,
+ );
+
+ expect(res.statusCode).toBe(400);
+ expect(res.body.success).toBe(false);
+ },
+ scrapeTimeout,
+ );
});
diff --git a/apps/api/src/controllers/v2/__tests__/gov-search.test.ts b/apps/api/src/controllers/v2/__tests__/gov-search.test.ts
index 6b80beaf4..c207d1a41 100644
--- a/apps/api/src/controllers/v2/__tests__/gov-search.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/gov-search.test.ts
@@ -188,6 +188,18 @@ describe("/v2/search/gov", () => {
["GET", { query: "rules", k: "101" }],
["GET", { query: "rules", magic: "true" }],
["POST", { query: "rules", magic: "true" }],
+ ["GET", { query: "rules", sources: "news" }],
+ ["POST", { query: "rules", sources: [] }],
+ ["POST", { query: "rules", sources: ["web", "news"] }],
+ ["POST", { query: "rules", sources: ["law"] }],
+ ["POST", { query: "rules", cites: [] }],
+ [
+ "POST",
+ {
+ query: "rules",
+ cites: Array.from({ length: 33 }, (_, i) => "us:usc:42:" + i),
+ },
+ ],
] as const)(
"rejects invalid %s input %j before the upstream call",
async (method, input) => {
@@ -203,6 +215,96 @@ describe("/v2/search/gov", () => {
},
);
+ it.each([
+ ["GET", ["gov", "gov"]],
+ ["GET", "gov"],
+ ["POST", ["gov"]],
+ ] as const)(
+ "forwards a %s request for gov results and counts them",
+ async (method, sources) => {
+ const labels = {
+ jurisdiction: "us",
+ court: "scotus",
+ cites: ["us:usc:21:343"],
+ lang: "en",
+ };
+ const gov = [
+ {
+ ...WEB_RESULT,
+ ...labels,
+ court_type: "federal-supreme",
+ cited_as: ["us:us-reports:555:223"],
+ updated_at: 1700000000000,
+ },
+ ];
+ mocks.fetchGovUpstream.mockResolvedValue({
+ ok: true,
+ status: 200,
+ headers: new Headers(),
+ text: async () => JSON.stringify({ success: true, data: { gov } }),
+ });
+ const res = makeRes();
+ await handler(method === "GET" ? "get" : "post")(
+ makeReq(method, { query: "food labeling rules", sources }),
+ res,
+ );
+ await flush();
+
+ expect(mocks.fetchGovUpstream).toHaveBeenCalledWith(
+ expect.objectContaining({ sources: ["gov"] }),
+ );
+ expect(res.json).toHaveBeenCalledWith({
+ success: true,
+ data: {
+ gov: [
+ {
+ ...WEB_RESULT,
+ ...labels,
+ courtType: "federal-supreme",
+ citedAs: ["us:us-reports:555:223"],
+ updatedAt: 1700000000000,
+ },
+ ],
+ },
+ });
+ expect(mocks.logResearchEndpoint).toHaveBeenCalledWith(
+ expect.objectContaining({ num_results: 1, credits_cost: 0 }),
+ );
+ },
+ );
+
+ it.each([
+ ["GET", "us:usc:42:1983", "us:usc:42:1983"],
+ [
+ "GET",
+ ["us:usc:42:1983", "us:us-reports:5:137"],
+ ["us:usc:42:1983", "us:us-reports:5:137"],
+ ],
+ ["POST", [" us:usc:42:1983 ", "us:usc:42:1983"], "us:usc:42:1983"],
+ ] as const)(
+ "forwards a %s cites filter %j as %j",
+ async (method, cites, expected) => {
+ const res = makeRes();
+ await handler(method === "GET" ? "get" : "post")(
+ makeReq(method, { query: "civil rights", cites }),
+ res,
+ );
+ await flush();
+
+ expect(mocks.fetchGovUpstream).toHaveBeenCalledWith(
+ expect.objectContaining({ cites: expected }),
+ );
+ },
+ );
+
+ it("leaves sources out of the upstream call unless sent", async () => {
+ const res = makeRes();
+ await handler("post")(makeReq("POST", { query: "zoning variance" }), res);
+ await flush();
+
+ expect(mocks.fetchGovUpstream.mock.calls[0][0].sources).toBeUndefined();
+ });
+
it("maps an upstream timeout to 504", async () => {
mocks.fetchGovUpstream.mockRejectedValue(
new DOMException("timed out", "TimeoutError"),
diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.ts
index d60110f8d..81d8af909 100644
--- a/apps/api/src/controllers/v2/research-proxy.ts
+++ b/apps/api/src/controllers/v2/research-proxy.ts
@@ -161,6 +161,12 @@ const DEVELOPER_SEARCH_QUERY_KEYS = [
"skills",
];
+const govSource = z.enum(["web", "gov"]);
+
+const govCite = z.string().trim().min(1);
+
+const govCites = z.array(govCite).min(1).max(32);
+
const govSearchSchema = z.strictObject({
query: z.string().min(1),
k: kSchema(100),
@@ -189,8 +195,31 @@ const govSearchSchema = z.strictObject({
"state-attorney-general",
])
.optional(),
- cites: z.string().trim().min(1).optional(),
+ cites: z
+ .unknown()
+ .transform((value, ctx) => {
+ const parsed = (Array.isArray(value) ? govCites : govCite).safeParse(
+ value,
+ );
+ if (!parsed.success) {
+ ctx.issues.push(
+ ...parsed.error.issues.map(
+ issue => ({ ...issue, input: value }) as z.core.$ZodRawIssue,
+ ),
+ );
+ return z.NEVER;
+ }
+ const cites = [...new Set([parsed.data].flat())];
+ return cites.length === 1 ? cites[0] : cites;
+ })
+ .optional(),
domain: z.string().trim().min(1).optional(),
+ sources: z
+ .union([govSource, z.array(govSource).min(1)])
+ .optional()
+ .transform(value =>
+ value === undefined ? undefined : [...new Set([value].flat())],
+ ),
...commonQuery,
});
@@ -267,9 +296,30 @@ function addLegacySnakeCaseAliases<T>(value: T): T {
return object as T;
}
+const GOV_RESULT_KEYS: Record<string, string> = {
+ court_type: "courtType",
+ cited_as: "citedAs",
+ updated_at: "updatedAt",
+};
+
+function camelCaseGovResults(body: any) {
+ const gov = body?.data?.gov;
+ if (!Array.isArray(gov)) return body;
+ body.data.gov = gov.map(item =>
+ Object.fromEntries(
+ Object.entries(item).map(([key, value]) => [
+ GOV_RESULT_KEYS[key] ?? key,
+ value,
+ ]),
+ ),
+ );
+ return body;
+}
+
function resultCount(body: any): number {
if (Array.isArray(body?.results)) return body.results.length;
- return Array.isArray(body?.data?.web) ? body.data.web.length : 0;
+ const group = body?.data?.web ?? body?.data?.gov;
+ return Array.isArray(group) ? group.length : 0;
}
function creditsFor(
@@ -508,10 +558,13 @@ function createResearchController(
if (responseBody === null || typeof responseBody === "string") {
return res.status(statusCode).send(responseBody ?? "");
}
- const response =
- options.legacy && upstream.ok
+ const response = !upstream.ok
+ ? responseBody
+ : options.legacy
? addLegacySnakeCaseAliases(responseBody)
- : responseBody;
+ : endpoint.kind === "gov_search"
+ ? camelCaseGovResults(responseBody)
+ : responseBody;
const usOnlyWarning =
upstream.ok && isUsOnly(authedReq.acuc?.flags)
? usOnlySkippedWarning(upstream.headers)
@@ -712,6 +765,7 @@ export function createGovRouter() {
courtType: params.courtType,
cites: params.cites,
domain: params.domain,
+ sources: params.sources,
headers,
timeoutMs: GOV_SEARCH_TIMEOUT_MS,
}),
diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.ts
index af095bdb2..ddb0b7c8c 100644
--- a/apps/api/src/lib/research-upstream.ts
+++ b/apps/api/src/lib/research-upstream.ts
@@ -121,8 +121,9 @@ export async function fetchGovUpstream(options: {
jurisdiction?: string;
court?: string;
courtType?: string;
- cites?: string;
+ cites?: string | string[];
domain?: string;
+ sources?: ("web" | "gov")[];
headers: Record<string, string>;
timeoutMs: number;
}) {
@@ -136,6 +137,7 @@ export async function fetchGovUpstream(options: {
court_type: options.courtType,
cites: options.cites,
domain: options.domain,
+ sources: options.sources,
},
options.headers,
options.timeoutMs,