Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add Government Index filters to /v2/search/gov and release the SDKs (#5338)
Kris Oeia3e4cc32ddec
diff --git a/apps/api/openapi.json b/apps/api/openapi.jsonindex 0eb072807..6da32eb5c 100644--- a/apps/api/openapi.json+++ b/apps/api/openapi.json@@ -6263,6 +6263,7 @@"get": {"summary": "Search the Government Index","operationId": "govSearch",+ "description": "Filters keep only pages the index has labeled, so a filtered search can leave out relevant pages, and no results does not mean none exist. Court labels are most complete for federal courts and case-law collections.","tags": ["Government"],"security": [{@@ -6291,6 +6292,71 @@"maximum": 100,"default": 10}+ },+ {+ "name": "jurisdiction",+ "in": "query",+ "required": false,+ "description": "Keep only pages of one jurisdiction: `us` for federal, or `us-` plus a two-letter state or territory code, such as `us-ca` or `us-dc`.",+ "schema": {+ "type": "string",+ "example": "us-ca"+ }+ },+ {+ "name": "court",+ "in": "query",+ "required": false,+ "description": "Keep only pages labeled for one court, by CourtListener court id, such as `scotus`, `ca9` or `nysd`.",+ "schema": {+ "type": "string",+ "minLength": 1,+ "example": "ca9"+ }+ },+ {+ "name": "courtType",+ "in": "query",+ "required": false,+ "description": "Keep only pages from one type of court.",+ "schema": {+ "type": "string",+ "enum": [+ "federal-supreme",+ "federal-appellate",+ "federal-district",+ "federal-bankruptcy",+ "federal-special",+ "military",+ "state-supreme",+ "state-appellate",+ "state-trial",+ "state-special",+ "state-attorney-general"+ ]+ }+ },+ {+ "name": "cites",+ "in": "query",+ "required": false,+ "description": "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`.",+ "schema": {+ "type": "string",+ "minLength": 1,+ "example": "42 U.S.C. § 1983"+ }+ },+ {+ "name": "domain",+ "in": "query",+ "required": false,+ "description": "Keep only pages from one site, such as `irs.gov`.",+ "schema": {+ "type": "string",+ "minLength": 1,+ "example": "irs.gov"+ }}],"responses": {@@ -6318,7 +6384,7 @@}},"400": {- "description": "Invalid request, including an unknown field"+ "description": "Invalid request, including an unknown field or an invalid filter value"},"401": {"description": "Invalid bearer token"@@ -6337,6 +6403,7 @@"post": {"summary": "Search the Government Index","operationId": "govSearchPost",+ "description": "Filters keep only pages the index has labeled, so a filtered search can leave out relevant pages, and no results does not mean none exist. Court labels are most complete for federal courts and case-law collections.","tags": ["Government"],"security": [{@@ -6363,6 +6430,46 @@"maximum": 100,"default": 10,"description": "Number of results to return."+ },+ "jurisdiction": {+ "type": "string",+ "description": "Keep only pages of one jurisdiction: `us` for federal, or `us-` plus a two-letter state or territory code, such as `us-ca` or `us-dc`.",+ "example": "us-ca"+ },+ "court": {+ "type": "string",+ "minLength": 1,+ "description": "Keep only pages labeled for one court, by CourtListener court id, such as `scotus`, `ca9` or `nysd`.",+ "example": "ca9"+ },+ "courtType": {+ "type": "string",+ "enum": [+ "federal-supreme",+ "federal-appellate",+ "federal-district",+ "federal-bankruptcy",+ "federal-special",+ "military",+ "state-supreme",+ "state-appellate",+ "state-trial",+ "state-special",+ "state-attorney-general"+ ],+ "description": "Keep only pages from one type of court."+ },+ "cites": {+ "type": "string",+ "minLength": 1,+ "description": "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`.",+ "example": "42 U.S.C. § 1983"+ },+ "domain": {+ "type": "string",+ "minLength": 1,+ "description": "Keep only pages from one site, such as `irs.gov`.",+ "example": "irs.gov"}}},@@ -6385,7 +6492,7 @@}},"400": {- "description": "Invalid request, including an unknown field"+ "description": "Invalid request, including an unknown field or an invalid filter value"},"401": {"description": "Invalid bearer token"diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 4745f87e6..99941d29f 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -157,6 +157,33 @@ const DEVELOPER_SEARCH_QUERY_KEYS = [const govSearchSchema = z.strictObject({query: z.string().min(1),k: kSchema(100),+ jurisdiction: z+ .string()+ .trim()+ .toLowerCase()+ .regex(/^us(-[a-z]{2})?$/, {+ message:+ 'jurisdiction must be "us" for federal, or "us-" plus a two-letter state or territory code, such as "us-ca" or "us-dc"',+ })+ .optional(),+ court: z.string().trim().min(1).optional(),+ courtType: z+ .enum([+ "federal-supreme",+ "federal-appellate",+ "federal-district",+ "federal-bankruptcy",+ "federal-special",+ "military",+ "state-supreme",+ "state-appellate",+ "state-trial",+ "state-special",+ "state-attorney-general",+ ])+ .optional(),+ cites: z.string().trim().min(1).optional(),+ domain: z.string().trim().min(1).optional(),...commonQuery,});@@ -443,6 +470,15 @@ function createResearchController(: undefined;}+ if (+ endpoint.kind === "gov_search" &&+ statusCode === 400 &&+ typeof responseBody?.message === "string"+ ) {+ error = responseBody.message;+ return researchError(res, 400, responseBody.message);+ }+if (responseBody === null || typeof responseBody === "string") {return res.status(statusCode).send(responseBody ?? "");}@@ -654,6 +690,11 @@ export function createGovRouter() {fetchGovUpstream({query: params.query,k: params.k,+ jurisdiction: params.jurisdiction,+ court: params.court,+ courtType: params.courtType,+ cites: params.cites,+ domain: params.domain,headers,timeoutMs: GOV_SEARCH_TIMEOUT_MS,}),diff --git a/apps/api/src/lib/research-upstream.ts b/apps/api/src/lib/research-upstream.tsindex 243b15ef5..cbb094224 100644--- a/apps/api/src/lib/research-upstream.ts+++ b/apps/api/src/lib/research-upstream.ts@@ -51,6 +51,11 @@ export async function fetchResearchUpstream(options: {export async function fetchGovUpstream(options: {query: string;k?: number;+ jurisdiction?: string;+ court?: string;+ courtType?: string;+ cites?: string;+ domain?: string;headers: Record<string, string>;timeoutMs: number;}) {@@ -60,7 +65,15 @@ export async function fetchGovUpstream(options: {return fetch(base.replace(/\/+$/, "") + "/api/v1/gov-search", {method: "POST",headers: { ...options.headers, "content-type": "application/json" },- body: JSON.stringify({ query: options.query, top_k: options.k }),+ body: JSON.stringify({+ query: options.query,+ top_k: options.k,+ jurisdiction: options.jurisdiction,+ court: options.court,+ court_type: options.courtType,+ cites: options.cites,+ domain: options.domain,+ }),signal: AbortSignal.timeout(options.timeoutMs),dispatcher,});diff --git a/apps/elixir-sdk/mix.exs b/apps/elixir-sdk/mix.exsindex a0b636dc8..ccbe5c12a 100644--- a/apps/elixir-sdk/mix.exs+++ b/apps/elixir-sdk/mix.exs@@ -1,7 +1,7 @@defmodule Firecrawl.MixProject douse Mix.Project- @version "1.12.4"+ @version "1.12.6"@source_url "https://github.com/firecrawl/firecrawl/tree/main/apps/elixir-sdk"def project dodiff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 4eec88e3d..e74f3ba42 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.50.1",+ "version": "4.51.0","description": "JavaScript SDK for Firecrawl, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv)","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 552abed5d..a0bb9a0be 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -42,7 +42,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.53.3"+__version__ = "4.54.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")