Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(sdks): pass gov search sources and type the gov result group (#5719)

Erik Engervall1fcb57c3e324
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 5f689608e..1f818ba29 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.53.0",
+ "version": "4.54.0",
"description": "JavaScript SDK for Firecrawl, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv)",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/gov.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/gov.test.ts
index 81153107e..1b212dc5e 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/gov.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/gov.test.ts
@@ -42,6 +42,40 @@ describe("govSearch", () => {
expect(result).toEqual(response);
});
+ test("sends sources and returns gov results with labels", async () => {
+ const labeled = {
+ success: true,
+ data: {
+ gov: [
+ {
+ ...response.data.web[0],
+ jurisdiction: "us",
+ courtType: "federal-appellate",
+ cites: ["us:usc:21:343"],
+ citedAs: ["us:cfr:21:101"],
+ updatedAt: 1700000000000,
+ },
+ ],
+ },
+ };
+ const http = {
+ post: vi.fn(async () => ({ status: 200, data: labeled })),
+ } as any;
+
+ const result = await clientWith(http).govSearch("food labeling", {
+ sources: ["gov"],
+ cites: ["21 U.S.C. § 343", "us:cfr:21:101"],
+ });
+
+ expect(http.post).toHaveBeenCalledWith("/v2/search/gov", {
+ query: "food labeling",
+ cites: ["21 U.S.C. § 343", "us:cfr:21:101"],
+ sources: ["gov"],
+ });
+ expect(result.data.gov?.[0].citedAs).toEqual(["us:cfr:21:101"]);
+ expect(result.data.web).toBeUndefined();
+ });
+
test("rejects an empty query", async () => {
const http = { post: vi.fn() } as any;
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/gov.ts b/apps/js-sdk/firecrawl/src/v2/methods/gov.ts
index f317ebb2d..3be1a0e56 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/gov.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/gov.ts
@@ -28,6 +28,7 @@ export async function govSearch(
: {}),
...(options.cites !== undefined ? { cites: options.cites } : {}),
...(options.domain !== undefined ? { domain: options.domain } : {}),
+ ...(options.sources !== undefined ? { sources: options.sources } : {}),
});
if (response.status !== 200 || !response.data?.success) {
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 34c1dba82..e32aedb98 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -856,15 +856,36 @@ export interface GovSearchOptions {
court?: string;
/** Keep only pages from one type of court. */
courtType?: GovSearchCourtType;
- /** Keep only pages that cite one law or case, as a printed citation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a citation id such as "us:usc:42:1983". */
- cites?: string;
+ /** Keep only pages that cite a law or case, as a printed citation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a citation id such as "us:usc:42:1983". A list of up to 32 keeps pages citing any of them. */
+ cites?: string | string[];
/** Keep only pages from one site, such as "irs.gov". */
domain?: string;
+ /** Result groups to return: `web` (the default) and `gov`, the same results with the index's labels. */
+ sources?: GovSearchSource[];
+}
+
+export type GovSearchSource = "web" | "gov";
+
+/** A Government Index result with the index's labels; a label the index does not have is absent. */
+export interface GovSearchResult extends SearchResultWeb {
+ /** "us" for federal, or "us-" plus a two-letter state or territory code. */
+ jurisdiction?: string;
+ /** CourtListener court id, such as "scotus" or "ca9". */
+ court?: string;
+ courtType?: GovSearchCourtType;
+ /** Citation ids of the laws and cases the page cites. */
+ cites?: string[];
+ /** Citation ids other pages cite this page by; pass one as `cites` to find them. */
+ citedAs?: string[];
+ lang?: string;
+ /** Crawl time in Unix milliseconds. */
+ updatedAt?: number;
}
export interface GovSearchResponse {
success: boolean;
- data: { web: SearchResultWeb[] };
+ /** `web` is present unless `sources` leaves it out; `gov` is present when `sources` includes it. */
+ data: { web?: SearchResultWeb[]; gov?: GovSearchResult[] };
}
export interface SearchResultWeb {
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 888f56eb7..9b5603d8e 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -49,7 +49,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.56.0"
+__version__ = "4.57.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_gov_search.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_gov_search.py
index 972c40ba1..6d5ef051c 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_gov_search.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_gov_search.py
@@ -52,6 +52,48 @@ def test_gov_search_posts_body_and_parses_web_results(kwargs, body):
assert result.data.web[0].position == 1
+def test_gov_search_sends_sources_and_parses_gov_results():
+ labeled = {
+ "success": True,
+ "data": {
+ "gov": [
+ {
+ **RESPONSE["data"]["web"][0],
+ "jurisdiction": "us",
+ "courtType": "federal-appellate",
+ "cites": ["us:usc:21:343"],
+ "citedAs": ["us:cfr:21:101"],
+ "updatedAt": 1700000000000,
+ }
+ ]
+ },
+ }
+ transport = Mock()
+ transport.post.return_value = _response(labeled)
+ client = FirecrawlClient.__new__(FirecrawlClient)
+ client.http_client = transport
+
+ result = client.gov_search(
+ "food labeling",
+ sources=["gov"],
+ cites=["21 U.S.C. § 343", "us:cfr:21:101"],
+ )
+
+ transport.post.assert_called_once_with(
+ "/v2/search/gov",
+ {
+ "query": "food labeling",
+ "cites": ["21 U.S.C. § 343", "us:cfr:21:101"],
+ "sources": ["gov"],
+ },
+ )
+ assert result.data.web is None
+ hit = result.data.gov[0]
+ assert hit.court_type == "federal-appellate"
+ assert hit.cited_as == ["us:cfr:21:101"]
+ assert hit.updated_at == 1700000000000
+
+
def test_gov_search_rejects_empty_query():
transport = Mock()
client = FirecrawlClient.__new__(FirecrawlClient)
diff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.py
index a3b385fb5..821ef2e18 100644
--- a/apps/python-sdk/firecrawl/types.py
+++ b/apps/python-sdk/firecrawl/types.py
@@ -70,7 +70,9 @@ from .v2.types import (
DeveloperSearchSourceStatus,
DeveloperSearchResponse,
GovSearchCourtType,
+ GovSearchSource,
GovSearchRequest,
+ GovSearchResult,
GovSearchData,
GovSearchResponse,
@@ -174,7 +176,9 @@ __all__ = [
'DeveloperSearchSourceStatus',
'DeveloperSearchResponse',
'GovSearchCourtType',
+ 'GovSearchSource',
'GovSearchRequest',
+ 'GovSearchResult',
'GovSearchData',
'GovSearchResponse',
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 5ba265aea..5cdfcb627 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -18,6 +18,7 @@ from .types import (
DeveloperSearchResponse,
DeveloperSearchType,
GovSearchCourtType,
+ GovSearchSource,
GovSearchResponse,
SourceOption,
CategoryOption,
@@ -627,8 +628,9 @@ class FirecrawlClient:
jurisdiction: Optional[str] = None,
court: Optional[str] = None,
court_type: Optional[GovSearchCourtType] = None,
- cites: Optional[str] = None,
+ cites: Optional[Union[str, List[str]]] = None,
domain: Optional[str] = None,
+ sources: Optional[List[GovSearchSource]] = None,
) -> GovSearchResponse:
"""
Search the Government Index of US primary law and regulatory material.
@@ -647,13 +649,17 @@ class FirecrawlClient:
id, such as "scotus", "ca9" or "nysd"
court_type: Keep only pages from one type of court, such as
"federal-appellate" or "state-supreme"
- cites: Keep only pages that cite one law or case, as a printed
+ cites: Keep only pages that cite a law or case, as a printed
citation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a
- citation id such as "us:usc:42:1983"
+ citation id such as "us:usc:42:1983"; a list of up to 32
+ keeps pages citing any of them
domain: Keep only pages from one site, such as "irs.gov"
+ sources: Result groups to return: "web" (the default) and "gov",
+ the same results with the index's labels
Returns:
- GovSearchResponse with ranked results in data.web
+ GovSearchResponse with ranked results in data.web (None when sources
+ is ["gov"]), and in data.gov when sources includes "gov"
"""
return gov_module.gov_search(
self.http_client,
@@ -664,6 +670,7 @@ class FirecrawlClient:
court_type=court_type,
cites=cites,
domain=domain,
+ sources=sources,
)
def crawl(
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index c4eb8bc21..dadbc48cc 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -21,6 +21,7 @@ from .types import (
DeveloperSearchResponse,
DeveloperSearchType,
GovSearchCourtType,
+ GovSearchSource,
GovSearchResponse,
SourceOption,
FindToolsData,
@@ -390,8 +391,9 @@ class AsyncFirecrawlClient:
jurisdiction: Optional[str] = None,
court: Optional[str] = None,
court_type: Optional[GovSearchCourtType] = None,
- cites: Optional[str] = None,
+ cites: Optional[Union[str, List[str]]] = None,
domain: Optional[str] = None,
+ sources: Optional[List[GovSearchSource]] = None,
) -> GovSearchResponse:
"""
Search the Government Index of US primary law and regulatory material.
@@ -410,13 +412,17 @@ class AsyncFirecrawlClient:
id, such as "scotus", "ca9" or "nysd"
court_type: Keep only pages from one type of court, such as
"federal-appellate" or "state-supreme"
- cites: Keep only pages that cite one law or case, as a printed
+ cites: Keep only pages that cite a law or case, as a printed
citation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a
- citation id such as "us:usc:42:1983"
+ citation id such as "us:usc:42:1983"; a list of up to 32
+ keeps pages citing any of them
domain: Keep only pages from one site, such as "irs.gov"
+ sources: Result groups to return: "web" (the default) and "gov",
+ the same results with the index's labels
Returns:
- GovSearchResponse with ranked results in data.web
+ GovSearchResponse with ranked results in data.web (None when sources
+ is ["gov"]), and in data.gov when sources includes "gov"
"""
return await async_gov.gov_search(
self.async_http_client,
@@ -427,6 +433,7 @@ class AsyncFirecrawlClient:
court_type=court_type,
cites=cites,
domain=domain,
+ sources=sources,
)
async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/gov.py b/apps/python-sdk/firecrawl/v2/methods/aio/gov.py
index 440a546d1..d7c104d33 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/gov.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/gov.py
@@ -1,8 +1,8 @@
"""Async Government Index search functionality for Firecrawl v2."""
-from typing import Optional
+from typing import List, Optional, Union
-from ...types import GovSearchCourtType, GovSearchRequest, GovSearchResponse
+from ...types import GovSearchCourtType, GovSearchRequest, GovSearchResponse, GovSearchSource
from ...utils.error_handler import handle_response_error
from ...utils.http_client_async import AsyncHttpClient
@@ -18,8 +18,9 @@ async def gov_search(
jurisdiction: Optional[str] = None,
court: Optional[str] = None,
court_type: Optional[GovSearchCourtType] = None,
- cites: Optional[str] = None,
+ cites: Optional[Union[str, List[str]]] = None,
domain: Optional[str] = None,
+ sources: Optional[List[GovSearchSource]] = None,
) -> GovSearchResponse:
"""Search the Government Index asynchronously."""
if not query or not query.strip():
@@ -33,6 +34,7 @@ async def gov_search(
court_type=court_type,
cites=cites,
domain=domain,
+ sources=sources,
)
response = await client.post(ENDPOINT, request.model_dump(exclude_none=True, by_alias=True))
if response.status_code != 200:
diff --git a/apps/python-sdk/firecrawl/v2/methods/gov.py b/apps/python-sdk/firecrawl/v2/methods/gov.py
index 2e2c8adb3..f65e22f0d 100644
--- a/apps/python-sdk/firecrawl/v2/methods/gov.py
+++ b/apps/python-sdk/firecrawl/v2/methods/gov.py
@@ -1,8 +1,8 @@
"""Government Index search functionality for Firecrawl v2."""
-from typing import Optional
+from typing import List, Optional, Union
-from ..types import GovSearchCourtType, GovSearchRequest, GovSearchResponse
+from ..types import GovSearchCourtType, GovSearchRequest, GovSearchResponse, GovSearchSource
from ..utils import HttpClient, handle_response_error
@@ -17,8 +17,9 @@ def gov_search(
jurisdiction: Optional[str] = None,
court: Optional[str] = None,
court_type: Optional[GovSearchCourtType] = None,
- cites: Optional[str] = None,
+ cites: Optional[Union[str, List[str]]] = None,
domain: Optional[str] = None,
+ sources: Optional[List[GovSearchSource]] = None,
) -> GovSearchResponse:
"""Search the Government Index."""
if not query or not query.strip():
@@ -32,6 +33,7 @@ def gov_search(
court_type=court_type,
cites=cites,
domain=domain,
+ sources=sources,
)
response = client.post(ENDPOINT, request.model_dump(exclude_none=True, by_alias=True))
if response.status_code != 200:
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 18f363f32..77fcd1a62 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -2545,6 +2545,8 @@ GovSearchCourtType = Literal[
"state-attorney-general",
]
+GovSearchSource = Literal["web", "gov"]
+
class GovSearchRequest(BaseModel):
"""Request for the Government Index search endpoint."""
@@ -2556,12 +2558,28 @@ class GovSearchRequest(BaseModel):
jurisdiction: Optional[str] = None
court: Optional[str] = None
court_type: Optional[GovSearchCourtType] = Field(default=None, alias="courtType")
- cites: Optional[str] = None
+ cites: Optional[Union[str, Annotated[List[str], Field(max_length=32)]]] = None
domain: Optional[str] = None
+ sources: Optional[List[GovSearchSource]] = None
+
+
+class GovSearchResult(SearchResultWeb):
+ """A Government Index result with the index's labels; a label the index does not have is None."""
+
+ model_config = {"populate_by_name": True}
+
+ jurisdiction: Optional[str] = None
+ court: Optional[str] = None
+ court_type: Optional[str] = Field(default=None, alias="courtType")
+ cites: Optional[List[str]] = None
+ cited_as: Optional[List[str]] = Field(default=None, alias="citedAs")
+ lang: Optional[str] = None
+ updated_at: Optional[int] = Field(default=None, alias="updatedAt")
class GovSearchData(BaseModel):
- web: List[SearchResultWeb]
+ web: Optional[List[SearchResultWeb]] = None
+ gov: Optional[List[GovSearchResult]] = None
class GovSearchResponse(BaseModel):