Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sdks): pass gov search sources and type the gov result group (#5719)
Erik Engervall1fcb57c3e324
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 5f689608e..1f818ba29 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.53.0",+ "version": "4.54.0","description": "JavaScript SDK for Firecrawl, the context API for AI agents: search, scrape, parse, crawl, map, and interact with the web, run Agent and Monitor jobs, and search scientific papers (PubMed, bioRxiv, medRxiv, arXiv)","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/gov.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/gov.test.tsindex 81153107e..1b212dc5e 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/gov.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/gov.test.ts@@ -42,6 +42,40 @@ describe("govSearch", () => {expect(result).toEqual(response);});+ test("sends sources and returns gov results with labels", async () => {+ const labeled = {+ success: true,+ data: {+ gov: [+ {+ ...response.data.web[0],+ jurisdiction: "us",+ courtType: "federal-appellate",+ cites: ["us:usc:21:343"],+ citedAs: ["us:cfr:21:101"],+ updatedAt: 1700000000000,+ },+ ],+ },+ };+ const http = {+ post: vi.fn(async () => ({ status: 200, data: labeled })),+ } as any;++ const result = await clientWith(http).govSearch("food labeling", {+ sources: ["gov"],+ cites: ["21 U.S.C. § 343", "us:cfr:21:101"],+ });++ expect(http.post).toHaveBeenCalledWith("/v2/search/gov", {+ query: "food labeling",+ cites: ["21 U.S.C. § 343", "us:cfr:21:101"],+ sources: ["gov"],+ });+ expect(result.data.gov?.[0].citedAs).toEqual(["us:cfr:21:101"]);+ expect(result.data.web).toBeUndefined();+ });+test("rejects an empty query", async () => {const http = { post: vi.fn() } as any;diff --git a/apps/js-sdk/firecrawl/src/v2/methods/gov.ts b/apps/js-sdk/firecrawl/src/v2/methods/gov.tsindex f317ebb2d..3be1a0e56 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/gov.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/gov.ts@@ -28,6 +28,7 @@ export async function govSearch(: {}),...(options.cites !== undefined ? { cites: options.cites } : {}),...(options.domain !== undefined ? { domain: options.domain } : {}),+ ...(options.sources !== undefined ? { sources: options.sources } : {}),});if (response.status !== 200 || !response.data?.success) {diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 34c1dba82..e32aedb98 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -856,15 +856,36 @@ export interface GovSearchOptions {court?: string;/** Keep only pages from one type of court. */courtType?: GovSearchCourtType;- /** Keep only pages that cite one law or case, as a printed citation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a citation id such as "us:usc:42:1983". */- cites?: string;+ /** Keep only pages that cite a law or case, as a printed citation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a citation id such as "us:usc:42:1983". A list of up to 32 keeps pages citing any of them. */+ cites?: string | string[];/** Keep only pages from one site, such as "irs.gov". */domain?: string;+ /** Result groups to return: `web` (the default) and `gov`, the same results with the index's labels. */+ sources?: GovSearchSource[];+}++export type GovSearchSource = "web" | "gov";++/** A Government Index result with the index's labels; a label the index does not have is absent. */+export interface GovSearchResult extends SearchResultWeb {+ /** "us" for federal, or "us-" plus a two-letter state or territory code. */+ jurisdiction?: string;+ /** CourtListener court id, such as "scotus" or "ca9". */+ court?: string;+ courtType?: GovSearchCourtType;+ /** Citation ids of the laws and cases the page cites. */+ cites?: string[];+ /** Citation ids other pages cite this page by; pass one as `cites` to find them. */+ citedAs?: string[];+ lang?: string;+ /** Crawl time in Unix milliseconds. */+ updatedAt?: number;}export interface GovSearchResponse {success: boolean;- data: { web: SearchResultWeb[] };+ /** `web` is present unless `sources` leaves it out; `gov` is present when `sources` includes it. */+ data: { web?: SearchResultWeb[]; gov?: GovSearchResult[] };}export interface SearchResultWeb {diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 888f56eb7..9b5603d8e 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -49,7 +49,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.56.0"+__version__ = "4.57.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_gov_search.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_gov_search.pyindex 972c40ba1..6d5ef051c 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_gov_search.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_gov_search.py@@ -52,6 +52,48 @@ def test_gov_search_posts_body_and_parses_web_results(kwargs, body):assert result.data.web[0].position == 1+def test_gov_search_sends_sources_and_parses_gov_results():+ labeled = {+ "success": True,+ "data": {+ "gov": [+ {+ **RESPONSE["data"]["web"][0],+ "jurisdiction": "us",+ "courtType": "federal-appellate",+ "cites": ["us:usc:21:343"],+ "citedAs": ["us:cfr:21:101"],+ "updatedAt": 1700000000000,+ }+ ]+ },+ }+ transport = Mock()+ transport.post.return_value = _response(labeled)+ client = FirecrawlClient.__new__(FirecrawlClient)+ client.http_client = transport++ result = client.gov_search(+ "food labeling",+ sources=["gov"],+ cites=["21 U.S.C. § 343", "us:cfr:21:101"],+ )++ transport.post.assert_called_once_with(+ "/v2/search/gov",+ {+ "query": "food labeling",+ "cites": ["21 U.S.C. § 343", "us:cfr:21:101"],+ "sources": ["gov"],+ },+ )+ assert result.data.web is None+ hit = result.data.gov[0]+ assert hit.court_type == "federal-appellate"+ assert hit.cited_as == ["us:cfr:21:101"]+ assert hit.updated_at == 1700000000000++def test_gov_search_rejects_empty_query():transport = Mock()client = FirecrawlClient.__new__(FirecrawlClient)diff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.pyindex a3b385fb5..821ef2e18 100644--- a/apps/python-sdk/firecrawl/types.py+++ b/apps/python-sdk/firecrawl/types.py@@ -70,7 +70,9 @@ from .v2.types import (DeveloperSearchSourceStatus,DeveloperSearchResponse,GovSearchCourtType,+ GovSearchSource,GovSearchRequest,+ GovSearchResult,GovSearchData,GovSearchResponse,@@ -174,7 +176,9 @@ __all__ = ['DeveloperSearchSourceStatus','DeveloperSearchResponse','GovSearchCourtType',+ 'GovSearchSource','GovSearchRequest',+ 'GovSearchResult','GovSearchData','GovSearchResponse',diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 5ba265aea..5cdfcb627 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -18,6 +18,7 @@ from .types import (DeveloperSearchResponse,DeveloperSearchType,GovSearchCourtType,+ GovSearchSource,GovSearchResponse,SourceOption,CategoryOption,@@ -627,8 +628,9 @@ class FirecrawlClient:jurisdiction: Optional[str] = None,court: Optional[str] = None,court_type: Optional[GovSearchCourtType] = None,- cites: Optional[str] = None,+ cites: Optional[Union[str, List[str]]] = None,domain: Optional[str] = None,+ sources: Optional[List[GovSearchSource]] = None,) -> GovSearchResponse:"""Search the Government Index of US primary law and regulatory material.@@ -647,13 +649,17 @@ class FirecrawlClient:id, such as "scotus", "ca9" or "nysd"court_type: Keep only pages from one type of court, such as"federal-appellate" or "state-supreme"- cites: Keep only pages that cite one law or case, as a printed+ cites: Keep only pages that cite a law or case, as a printedcitation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a- citation id such as "us:usc:42:1983"+ citation id such as "us:usc:42:1983"; a list of up to 32+ keeps pages citing any of themdomain: Keep only pages from one site, such as "irs.gov"+ sources: Result groups to return: "web" (the default) and "gov",+ the same results with the index's labelsReturns:- GovSearchResponse with ranked results in data.web+ GovSearchResponse with ranked results in data.web (None when sources+ is ["gov"]), and in data.gov when sources includes "gov""""return gov_module.gov_search(self.http_client,@@ -664,6 +670,7 @@ class FirecrawlClient:court_type=court_type,cites=cites,domain=domain,+ sources=sources,)def crawl(diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex c4eb8bc21..dadbc48cc 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -21,6 +21,7 @@ from .types import (DeveloperSearchResponse,DeveloperSearchType,GovSearchCourtType,+ GovSearchSource,GovSearchResponse,SourceOption,FindToolsData,@@ -390,8 +391,9 @@ class AsyncFirecrawlClient:jurisdiction: Optional[str] = None,court: Optional[str] = None,court_type: Optional[GovSearchCourtType] = None,- cites: Optional[str] = None,+ cites: Optional[Union[str, List[str]]] = None,domain: Optional[str] = None,+ sources: Optional[List[GovSearchSource]] = None,) -> GovSearchResponse:"""Search the Government Index of US primary law and regulatory material.@@ -410,13 +412,17 @@ class AsyncFirecrawlClient:id, such as "scotus", "ca9" or "nysd"court_type: Keep only pages from one type of court, such as"federal-appellate" or "state-supreme"- cites: Keep only pages that cite one law or case, as a printed+ cites: Keep only pages that cite a law or case, as a printedcitation such as "42 U.S.C. § 1983" or "5 U.S. 137", or a- citation id such as "us:usc:42:1983"+ citation id such as "us:usc:42:1983"; a list of up to 32+ keeps pages citing any of themdomain: Keep only pages from one site, such as "irs.gov"+ sources: Result groups to return: "web" (the default) and "gov",+ the same results with the index's labelsReturns:- GovSearchResponse with ranked results in data.web+ GovSearchResponse with ranked results in data.web (None when sources+ is ["gov"]), and in data.gov when sources includes "gov""""return await async_gov.gov_search(self.async_http_client,@@ -427,6 +433,7 @@ class AsyncFirecrawlClient:court_type=court_type,cites=cites,domain=domain,+ sources=sources,)async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/gov.py b/apps/python-sdk/firecrawl/v2/methods/aio/gov.pyindex 440a546d1..d7c104d33 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/gov.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/gov.py@@ -1,8 +1,8 @@"""Async Government Index search functionality for Firecrawl v2."""-from typing import Optional+from typing import List, Optional, Union-from ...types import GovSearchCourtType, GovSearchRequest, GovSearchResponse+from ...types import GovSearchCourtType, GovSearchRequest, GovSearchResponse, GovSearchSourcefrom ...utils.error_handler import handle_response_errorfrom ...utils.http_client_async import AsyncHttpClient@@ -18,8 +18,9 @@ async def gov_search(jurisdiction: Optional[str] = None,court: Optional[str] = None,court_type: Optional[GovSearchCourtType] = None,- cites: Optional[str] = None,+ cites: Optional[Union[str, List[str]]] = None,domain: Optional[str] = None,+ sources: Optional[List[GovSearchSource]] = None,) -> GovSearchResponse:"""Search the Government Index asynchronously."""if not query or not query.strip():@@ -33,6 +34,7 @@ async def gov_search(court_type=court_type,cites=cites,domain=domain,+ sources=sources,)response = await client.post(ENDPOINT, request.model_dump(exclude_none=True, by_alias=True))if response.status_code != 200:diff --git a/apps/python-sdk/firecrawl/v2/methods/gov.py b/apps/python-sdk/firecrawl/v2/methods/gov.pyindex 2e2c8adb3..f65e22f0d 100644--- a/apps/python-sdk/firecrawl/v2/methods/gov.py+++ b/apps/python-sdk/firecrawl/v2/methods/gov.py@@ -1,8 +1,8 @@"""Government Index search functionality for Firecrawl v2."""-from typing import Optional+from typing import List, Optional, Union-from ..types import GovSearchCourtType, GovSearchRequest, GovSearchResponse+from ..types import GovSearchCourtType, GovSearchRequest, GovSearchResponse, GovSearchSourcefrom ..utils import HttpClient, handle_response_error@@ -17,8 +17,9 @@ def gov_search(jurisdiction: Optional[str] = None,court: Optional[str] = None,court_type: Optional[GovSearchCourtType] = None,- cites: Optional[str] = None,+ cites: Optional[Union[str, List[str]]] = None,domain: Optional[str] = None,+ sources: Optional[List[GovSearchSource]] = None,) -> GovSearchResponse:"""Search the Government Index."""if not query or not query.strip():@@ -32,6 +33,7 @@ def gov_search(court_type=court_type,cites=cites,domain=domain,+ sources=sources,)response = client.post(ENDPOINT, request.model_dump(exclude_none=True, by_alias=True))if response.status_code != 200:diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 18f363f32..77fcd1a62 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -2545,6 +2545,8 @@ GovSearchCourtType = Literal["state-attorney-general",]+GovSearchSource = Literal["web", "gov"]+class GovSearchRequest(BaseModel):"""Request for the Government Index search endpoint."""@@ -2556,12 +2558,28 @@ class GovSearchRequest(BaseModel):jurisdiction: Optional[str] = Nonecourt: Optional[str] = Nonecourt_type: Optional[GovSearchCourtType] = Field(default=None, alias="courtType")- cites: Optional[str] = None+ cites: Optional[Union[str, Annotated[List[str], Field(max_length=32)]]] = Nonedomain: Optional[str] = None+ sources: Optional[List[GovSearchSource]] = None+++class GovSearchResult(SearchResultWeb):+ """A Government Index result with the index's labels; a label the index does not have is None."""++ model_config = {"populate_by_name": True}++ jurisdiction: Optional[str] = None+ court: Optional[str] = None+ court_type: Optional[str] = Field(default=None, alias="courtType")+ cites: Optional[List[str]] = None+ cited_as: Optional[List[str]] = Field(default=None, alias="citedAs")+ lang: Optional[str] = None+ updated_at: Optional[int] = Field(default=None, alias="updatedAt")class GovSearchData(BaseModel):- web: List[SearchResultWeb]+ web: Optional[List[SearchResultWeb]] = None+ gov: Optional[List[GovSearchResult]] = Noneclass GovSearchResponse(BaseModel):