Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sdks): add Government Index filters to gov search (#5339)
Kris Oeia5bc4e9eef81
diff --git a/apps/elixir-sdk/lib/firecrawl.ex b/apps/elixir-sdk/lib/firecrawl.exindex 2fb03e760..3b481f3e3 100644--- a/apps/elixir-sdk/lib/firecrawl.ex+++ b/apps/elixir-sdk/lib/firecrawl.ex@@ -1675,10 +1675,15 @@ defmodule Firecrawl do@gov_search_query_schema NimbleOptions.new!([query: [type: :string, required: true, doc: "Natural-language question or search phrase."],- k: [type: :integer, doc: "Number of results to return."]+ k: [type: :integer, doc: "Number of results to return."],+ jurisdiction: [type: :string, doc: "Keep only pages of one jurisdiction: `us` for federal, or `us-` plus a two-letter state or territory code, such as `us-ca` or `us-dc`."],+ court: [type: :string, doc: "Keep only pages labeled for one court, by CourtListener court id, such as `scotus`, `ca9` or `nysd`."],+ court_type: [type: {:or, [{:in, [:"federal-supreme", :"federal-appellate", :"federal-district", :"federal-bankruptcy", :"federal-special", :military, :"state-supreme", :"state-appellate", :"state-trial", :"state-special", :"state-attorney-general"]}, :string]}, doc: "Keep only pages from one type of court: `federal-supreme`, `federal-appellate`, `federal-district`, `federal-bankruptcy`, `federal-special`, `military`, `state-supreme`, `state-appellate`, `state-trial`, `state-special`, `state-attorney-general`."],+ cites: [type: :string, doc: "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`."],+ domain: [type: :string, doc: "Keep only pages from one site, such as `irs.gov`."]])- @gov_search_query_key_mapping %{query: "query", k: "k"}+ @gov_search_query_key_mapping %{query: "query", k: "k", jurisdiction: "jurisdiction", court: "court", court_type: "courtType", cites: "cites", domain: "domain"}@doc """Search the Government Index@@ -1691,6 +1696,11 @@ defmodule Firecrawl do* `query` — query parameter `query`* `k` — query parameter `k`+ * `jurisdiction` — query parameter `jurisdiction`+ * `court` — query parameter `court`+ * `court_type` — query parameter `courtType`+ * `cites` — query parameter `cites`+ * `domain` — query parameter `domain`## Returns@@ -1716,11 +1726,16 @@ defmodule Firecrawl do@gov_search_post_schema NimbleOptions.new!([+ cites: [type: :string, doc: "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`."],+ court: [type: :string, doc: "Keep only pages labeled for one court, by CourtListener court id, such as `scotus`, `ca9` or `nysd`."],+ court_type: [type: {:or, [{:in, [:"federal-supreme", :"federal-appellate", :"federal-district", :"federal-bankruptcy", :"federal-special", :military, :"state-supreme", :"state-appellate", :"state-trial", :"state-special", :"state-attorney-general"]}, :string]}, doc: "Keep only pages from one type of court: `federal-supreme`, `federal-appellate`, `federal-district`, `federal-bankruptcy`, `federal-special`, `military`, `state-supreme`, `state-appellate`, `state-trial`, `state-special`, `state-attorney-general`."],+ domain: [type: :string, doc: "Keep only pages from one site, such as `irs.gov`."],+ jurisdiction: [type: :string, doc: "Keep only pages of one jurisdiction: `us` for federal, or `us-` plus a two-letter state or territory code, such as `us-ca` or `us-dc`."],k: [type: :integer, doc: "Number of results to return."],query: [type: :string, required: true, doc: "Natural-language question or search phrase."]])- @gov_search_post_key_mapping %{k: "k", query: "query"}+ @gov_search_post_key_mapping %{cites: "cites", court: "court", court_type: "courtType", domain: "domain", jurisdiction: "jurisdiction", k: "k", query: "query"}@doc """Search the Government Indexdiff --git a/apps/elixir-sdk/openapi.json b/apps/elixir-sdk/openapi.jsonindex d3534172d..9b900b49b 100644--- a/apps/elixir-sdk/openapi.json+++ b/apps/elixir-sdk/openapi.json@@ -6618,6 +6618,68 @@"maximum": 100,"default": 10}+ },+ {+ "name": "jurisdiction",+ "in": "query",+ "required": false,+ "description": "Keep only pages of one jurisdiction: `us` for federal, or `us-` plus a two-letter state or territory code, such as `us-ca` or `us-dc`.",+ "schema": {+ "type": "string",+ "pattern": "^us(-[a-zA-Z]{2})?$"+ }+ },+ {+ "name": "court",+ "in": "query",+ "required": false,+ "description": "Keep only pages labeled for one court, by CourtListener court id, such as `scotus`, `ca9` or `nysd`.",+ "schema": {+ "type": "string",+ "minLength": 1+ }+ },+ {+ "name": "courtType",+ "in": "query",+ "required": false,+ "description": "Keep only pages from one type of court: `federal-supreme`, `federal-appellate`, `federal-district`, `federal-bankruptcy`, `federal-special`, `military`, `state-supreme`, `state-appellate`, `state-trial`, `state-special`, `state-attorney-general`.",+ "schema": {+ "type": "string",+ "enum": [+ "federal-supreme",+ "federal-appellate",+ "federal-district",+ "federal-bankruptcy",+ "federal-special",+ "military",+ "state-supreme",+ "state-appellate",+ "state-trial",+ "state-special",+ "state-attorney-general"+ ]+ }+ },+ {+ "name": "cites",+ "in": "query",+ "required": false,+ "description": "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`.",+ "schema": {+ "type": "string",+ "minLength": 1+ }+ },+ {+ "name": "domain",+ "in": "query",+ "required": false,+ "description": "Keep only pages from one site, such as `irs.gov`.",+ "schema": {+ "type": "string",+ "minLength": 1+ }}],"responses": {@@ -6694,6 +6756,43 @@"maximum": 100,"default": 10,"description": "Number of results to return."+ },+ "jurisdiction": {+ "type": "string",+ "description": "Keep only pages of one jurisdiction: `us` for federal, or `us-` plus a two-letter state or territory code, such as `us-ca` or `us-dc`.",+ "pattern": "^us(-[a-zA-Z]{2})?$"+ },+ "court": {+ "type": "string",+ "minLength": 1,+ "description": "Keep only pages labeled for one court, by CourtListener court id, such as `scotus`, `ca9` or `nysd`."+ },+ "courtType": {+ "type": "string",+ "enum": [+ "federal-supreme",+ "federal-appellate",+ "federal-district",+ "federal-bankruptcy",+ "federal-special",+ "military",+ "state-supreme",+ "state-appellate",+ "state-trial",+ "state-special",+ "state-attorney-general"+ ],+ "description": "Keep only pages from one type of court: `federal-supreme`, `federal-appellate`, `federal-district`, `federal-bankruptcy`, `federal-special`, `military`, `state-supreme`, `state-appellate`, `state-trial`, `state-special`, `state-attorney-general`."+ },+ "cites": {+ "type": "string",+ "minLength": 1,+ "description": "Keep only pages that cite one law or case, as a printed citation such as `42 U.S.C. § 1983` or `5 U.S. 137`, or a citation id such as `us:usc:42:1983`."+ },+ "domain": {+ "type": "string",+ "minLength": 1,+ "description": "Keep only pages from one site, such as `irs.gov`."}}},diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex fed0e7afd..625aa8b75 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -264,8 +264,23 @@ const law = await app.govSearch('food labeling requirements', {for (const result of law.data.web) {console.log(result.position, result.title, result.url);}++// State supreme court opinions that cite a federal statute:+const cases = await app.govSearch('qualified immunity', {+ courtType: 'state-supreme',+ cites: '42 U.S.C. § 1983',+});```+Filters, all optional and combined with AND: `jurisdiction` (`'us'` for+federal, or `'us-'` plus a two-letter state or territory code, such as+`'us-ca'`), `court` (a CourtListener court id, such as `'scotus'` or `'ca9'`),+`courtType`, `cites` (one printed citation or citation id, such as+`'us:usc:42:1983'`), and `domain` (one site, such as `'irs.gov'`). Filters keep+only pages the index has labeled, so a filtered search can leave out relevant+pages, and no results does not mean none exist. Court labels are most complete+for federal courts and case-law collections.+### Research / paper searchUse `app.research` to search Firecrawl's research paper index: ~43M paperdiff --git a/apps/js-sdk/firecrawl/src/v2/methods/gov.ts b/apps/js-sdk/firecrawl/src/v2/methods/gov.tsindex 8717b1297..f317ebb2d 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/gov.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/gov.ts@@ -19,6 +19,15 @@ export async function govSearch(const response = await http.post<GovSearchResponse>(ENDPOINT, {query,...(options.k !== undefined ? { k: options.k } : {}),+ ...(options.jurisdiction !== undefined+ ? { jurisdiction: options.jurisdiction }+ : {}),+ ...(options.court !== undefined ? { court: options.court } : {}),+ ...(options.courtType !== undefined+ ? { courtType: options.courtType }+ : {}),+ ...(options.cites !== undefined ? { cites: options.cites } : {}),+ ...(options.domain !== undefined ? { domain: options.domain } : {}),});if (response.status !== 200 || !response.data?.success) {diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 145ff74c2..4c4675284 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -831,9 +831,32 @@ export interface DeveloperSearchResponse {sources?: DeveloperSearchSourceStatus[];}+export type GovSearchCourtType =+ | "federal-supreme"+ | "federal-appellate"+ | "federal-district"+ | "federal-bankruptcy"+ | "federal-special"+ | "military"+ | "state-supreme"+ | "state-appellate"+ | "state-trial"+ | "state-special"+ | "state-attorney-general";+export interface GovSearchOptions {/** Total ranked results, 1–100 (default 10). */k?: number;+ /** Keep only pages of one jurisdiction: "us" for federal, or "us-" plus a two-letter state or territory code, such as "us-ca" or "us-dc". */+ jurisdiction?: string;+ /** Keep only pages labeled for one court, by CourtListener court id, such as "scotus", "ca9" or "nysd". */+ court?: string;+ /** Keep only pages from one type of court. */+ courtType?: GovSearchCourtType;+ /** Keep only pages that cite one law or case, as a printed citation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a citation id such as "us:usc:42:1983". */+ cites?: string;+ /** Keep only pages from one site, such as "irs.gov". */+ domain?: string;}export interface GovSearchResponse {diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex 2621ef8ae..fc9cb82ea 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -285,8 +285,24 @@ law = firecrawl.gov_search("food labeling requirements", k=5)for result in law.data.web:print(result.position, result.title, result.url)++# State supreme court opinions that cite a federal statute:+cases = firecrawl.gov_search(+ "qualified immunity",+ court_type="state-supreme",+ cites="42 U.S.C. § 1983",+)```+Filters, all optional and combined with AND: `jurisdiction` (`"us"` for+federal, or `"us-"` plus a two-letter state or territory code, such as+`"us-ca"`), `court` (a CourtListener court id, such as `"scotus"` or `"ca9"`),+`court_type`, `cites` (one printed citation or citation id, such as+`"us:usc:42:1983"`), and `domain` (one site, such as `"irs.gov"`). Filters+keep only pages the index has labeled, so a filtered search can leave out+relevant pages, and no results does not mean none exist. Court labels are most+complete for federal courts and case-law collections.+### Research / paper searchUse `search_papers` to search Firecrawl's research paper index: ~43M paperdiff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.pyindex ff39f6da3..a3b385fb5 100644--- a/apps/python-sdk/firecrawl/types.py+++ b/apps/python-sdk/firecrawl/types.py@@ -69,6 +69,7 @@ from .v2.types import (DeveloperSearchRepoStatus,DeveloperSearchSourceStatus,DeveloperSearchResponse,+ GovSearchCourtType,GovSearchRequest,GovSearchData,GovSearchResponse,@@ -172,6 +173,7 @@ __all__ = ['DeveloperSearchRepoStatus','DeveloperSearchSourceStatus','DeveloperSearchResponse',+ 'GovSearchCourtType','GovSearchRequest','GovSearchData','GovSearchResponse',diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 8091a9df3..95d85164c 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -17,6 +17,7 @@ from .types import (SearchData,DeveloperSearchResponse,DeveloperSearchType,+ GovSearchCourtType,GovSearchResponse,SourceOption,CategoryOption,@@ -599,10 +600,47 @@ class FirecrawlClient:self,query: str,k: Optional[int] = None,+ *,+ jurisdiction: Optional[str] = None,+ court: Optional[str] = None,+ court_type: Optional[GovSearchCourtType] = None,+ cites: Optional[str] = None,+ domain: Optional[str] = None,) -> GovSearchResponse:- """Search the Government Index of US primary law and regulatory material."""+ """+ Search the Government Index of US primary law and regulatory material.++ Filters keep only pages the index has labeled, so a filtered search can+ leave out relevant pages, and no results does not mean none exist. Court+ labels are most complete for federal courts and case-law collections.++ Args:+ query: Search query string+ k: Number of results to return, 1-100 (default: 10)+ jurisdiction: Keep only pages of one jurisdiction: "us" for federal,+ or "us-" plus a two-letter state or territory code, such as+ "us-ca" or "us-dc"+ court: Keep only pages labeled for one court, by CourtListener court+ id, such as "scotus", "ca9" or "nysd"+ court_type: Keep only pages from one type of court, such as+ "federal-appellate" or "state-supreme"+ cites: Keep only pages that cite one law or case, as a printed+ citation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a+ citation id such as "us:usc:42:1983"+ domain: Keep only pages from one site, such as "irs.gov"++ Returns:+ GovSearchResponse with ranked results in data.web+ """return gov_module.gov_search(- self.http_client, query, k=k+ self.http_client,+ query,+ k=k,+ jurisdiction=jurisdiction,+ court=court,+ court_type=court_type,+ cites=cites,+ domain=domain,)def crawl(diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex a8de23256..0db2e7c8b 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -20,6 +20,7 @@ from .types import (SearchData,DeveloperSearchResponse,DeveloperSearchType,+ GovSearchCourtType,GovSearchResponse,SourceOption,FindToolsData,@@ -368,10 +369,47 @@ class AsyncFirecrawlClient:self,query: str,k: Optional[int] = None,+ *,+ jurisdiction: Optional[str] = None,+ court: Optional[str] = None,+ court_type: Optional[GovSearchCourtType] = None,+ cites: Optional[str] = None,+ domain: Optional[str] = None,) -> GovSearchResponse:- """Search the Government Index of US primary law and regulatory material."""+ """+ Search the Government Index of US primary law and regulatory material.++ Filters keep only pages the index has labeled, so a filtered search can+ leave out relevant pages, and no results does not mean none exist. Court+ labels are most complete for federal courts and case-law collections.++ Args:+ query: Search query string+ k: Number of results to return, 1-100 (default: 10)+ jurisdiction: Keep only pages of one jurisdiction: "us" for federal,+ or "us-" plus a two-letter state or territory code, such as+ "us-ca" or "us-dc"+ court: Keep only pages labeled for one court, by CourtListener court+ id, such as "scotus", "ca9" or "nysd"+ court_type: Keep only pages from one type of court, such as+ "federal-appellate" or "state-supreme"+ cites: Keep only pages that cite one law or case, as a printed+ citation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a+ citation id such as "us:usc:42:1983"+ domain: Keep only pages from one site, such as "irs.gov"++ Returns:+ GovSearchResponse with ranked results in data.web+ """return await async_gov.gov_search(- self.async_http_client, query, k=k+ self.async_http_client,+ query,+ k=k,+ jurisdiction=jurisdiction,+ court=court,+ court_type=court_type,+ cites=cites,+ domain=domain,)async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/gov.py b/apps/python-sdk/firecrawl/v2/methods/aio/gov.pyindex cfcc30e16..440a546d1 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/gov.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/gov.py@@ -2,7 +2,7 @@from typing import Optional-from ...types import GovSearchRequest, GovSearchResponse+from ...types import GovSearchCourtType, GovSearchRequest, GovSearchResponsefrom ...utils.error_handler import handle_response_errorfrom ...utils.http_client_async import AsyncHttpClient@@ -15,13 +15,26 @@ async def gov_search(query: str,*,k: Optional[int] = None,+ jurisdiction: Optional[str] = None,+ court: Optional[str] = None,+ court_type: Optional[GovSearchCourtType] = None,+ cites: Optional[str] = None,+ domain: Optional[str] = None,) -> GovSearchResponse:"""Search the Government Index asynchronously."""if not query or not query.strip():raise ValueError("query cannot be empty")- request = GovSearchRequest(query=query, k=k)- response = await client.post(ENDPOINT, request.model_dump(exclude_none=True))+ request = GovSearchRequest(+ query=query,+ k=k,+ jurisdiction=jurisdiction,+ court=court,+ court_type=court_type,+ cites=cites,+ domain=domain,+ )+ response = await client.post(ENDPOINT, request.model_dump(exclude_none=True, by_alias=True))if response.status_code != 200:handle_response_error(response, "search the Government Index")response_data = response.json()diff --git a/apps/python-sdk/firecrawl/v2/methods/gov.py b/apps/python-sdk/firecrawl/v2/methods/gov.pyindex e06d0041b..2e2c8adb3 100644--- a/apps/python-sdk/firecrawl/v2/methods/gov.py+++ b/apps/python-sdk/firecrawl/v2/methods/gov.py@@ -2,7 +2,7 @@from typing import Optional-from ..types import GovSearchRequest, GovSearchResponse+from ..types import GovSearchCourtType, GovSearchRequest, GovSearchResponsefrom ..utils import HttpClient, handle_response_error@@ -14,13 +14,26 @@ def gov_search(query: str,*,k: Optional[int] = None,+ jurisdiction: Optional[str] = None,+ court: Optional[str] = None,+ court_type: Optional[GovSearchCourtType] = None,+ cites: Optional[str] = None,+ domain: Optional[str] = None,) -> GovSearchResponse:"""Search the Government Index."""if not query or not query.strip():raise ValueError("query cannot be empty")- request = GovSearchRequest(query=query, k=k)- response = client.post(ENDPOINT, request.model_dump(exclude_none=True))+ request = GovSearchRequest(+ query=query,+ k=k,+ jurisdiction=jurisdiction,+ court=court,+ court_type=court_type,+ cites=cites,+ domain=domain,+ )+ response = client.post(ENDPOINT, request.model_dump(exclude_none=True, by_alias=True))if response.status_code != 200:handle_response_error(response, "search the Government Index")response_data = response.json()diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 057f780bb..111f35cd1 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -2429,11 +2429,33 @@ class DeveloperSearchResponse(BaseModel):sources: Optional[List[DeveloperSearchSourceStatus]] = None+GovSearchCourtType = Literal[+ "federal-supreme",+ "federal-appellate",+ "federal-district",+ "federal-bankruptcy",+ "federal-special",+ "military",+ "state-supreme",+ "state-appellate",+ "state-trial",+ "state-special",+ "state-attorney-general",+]++class GovSearchRequest(BaseModel):"""Request for the Government Index search endpoint."""+ model_config = {"populate_by_name": True}+query: strk: Optional[int] = Field(default=None, ge=1, le=100)+ jurisdiction: Optional[str] = None+ court: Optional[str] = None+ court_type: Optional[GovSearchCourtType] = Field(default=None, alias="courtType")+ cites: Optional[str] = None+ domain: Optional[str] = Noneclass GovSearchData(BaseModel):