Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sdk): add an index search method
Kris Oei564ac3f353e8
diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex 5d2e3ef34..3e8fa9e79 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -236,6 +236,23 @@ console.log(evidence.repos); // indexed-status echoes for requested repos`archived`, `fork`, and `skills: 'only'`. Supplying both `repos` and `sources`OR-combines GitHub-backed and documentation results.+### Legal and regulatory search++Use `legalRegulatorySearch` to search the Legal and Regulatory Index: primary+law and regulatory material from US federal, state, and local government+sources, including statutes, regulations, codes, court opinions, and other+government publications. Results come back in the ordinary web-result shape.++```js+const law = await app.legalRegulatorySearch('food labeling requirements', {+ k: 5,+});++for (const result of law.data.web) {+ console.log(result.position, result.title, result.url);+}+```+### Research / paper searchUse `app.research` to search Firecrawl's research paper index: ~43M paperdiff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/legal-regulatory.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/legal-regulatory.test.tsnew file mode 100644index 000000000..981bc722e--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/legal-regulatory.test.ts@@ -0,0 +1,82 @@+import { describe, expect, jest, test } from "@jest/globals";+import { FirecrawlClient } from "../../../v2/client";+import { SdkError } from "../../../v2/types";++const response = {+ success: true,+ data: {+ web: [+ {+ url: "https://www.ecfr.gov/current/title-21/chapter-I/subchapter-B/part-101",+ title: "21 CFR Part 101 -- Food Labeling",+ description: "matched snippet",+ position: 1,+ },+ ],+ },+};++function clientWith(http: any) {+ const client = new FirecrawlClient({+ apiKey: "test",+ apiUrl: "http://localhost",+ });+ (client as any).http = http;+ return client;+}++describe("legalRegulatorySearch", () => {+ test("posts query and k and returns web results", async () => {+ const http = {+ post: jest.fn(async () => ({ status: 200, data: response })),+ } as any;++ const result = await clientWith(http).legalRegulatorySearch(+ "food labeling requirements",+ { k: 5 },+ );++ expect(http.post).toHaveBeenCalledWith("/v2/search/legal-regulatory", {+ query: "food labeling requirements",+ k: 5,+ });+ expect(result).toEqual(response);+ });++ test("omits k when not provided", async () => {+ const http = {+ post: jest.fn(async () => ({ status: 200, data: response })),+ } as any;++ await clientWith(http).legalRegulatorySearch("zoning variance");++ expect(http.post).toHaveBeenCalledWith("/v2/search/legal-regulatory", {+ query: "zoning variance",+ });+ });++ test("rejects an empty query", async () => {+ const http = { post: jest.fn() } as any;++ await expect(+ clientWith(http).legalRegulatorySearch(" "),+ ).rejects.toThrow("query cannot be empty");+ expect(http.post).not.toHaveBeenCalled();+ });++ test("normalizes transport errors to SdkError", async () => {+ const http = {+ post: jest.fn(async () => {+ throw {+ isAxiosError: true,+ code: "ECONNABORTED",+ message: "request timed out",+ };+ }),+ } as any;++ await expect(+ clientWith(http).legalRegulatorySearch("network failure"),+ ).rejects.toBeInstanceOf(SdkError);+ });+});diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex 8554c7d7d..975b034cd 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -8,6 +8,7 @@ import { parse as parseMethod, getParseFormats } from "./methods/parse";import { search } from "./methods/search";import { scrapeAlexandria, findTools } from "./methods/tools";import { developerSearch as developerSearchMethod } from "./methods/developer";+import { legalRegulatorySearch as legalRegulatorySearchMethod } from "./methods/legalRegulatory";import { map as mapMethod } from "./methods/map";import { feedback as feedbackMethod, searchFeedback as searchFeedbackMethod } from "./methods/feedback";import {@@ -60,6 +61,8 @@ import type {SearchRequest,DeveloperSearchOptions,DeveloperSearchResponse,+ LegalRegulatorySearchOptions,+ LegalRegulatorySearchResponse,EndpointFeedbackRequest,FeedbackResponse,SearchFeedbackRequest,@@ -306,6 +309,17 @@ export class FirecrawlClient {return developerSearchMethod(this.http, query, options);}+ /**+ * Search the Legal and Regulatory Index: primary law and regulatory material+ * from US federal, state, and local government sources.+ */+ async legalRegulatorySearch(+ query: string,+ options: LegalRegulatorySearchOptions = {},+ ): Promise<LegalRegulatorySearchResponse> {+ return legalRegulatorySearchMethod(this.http, query, options);+ }+/*** Submit feedback for a v2 job.* @param request Feedback payload with endpoint, job id, rating, and supporting signals.diff --git a/apps/js-sdk/firecrawl/src/v2/methods/legalRegulatory.ts b/apps/js-sdk/firecrawl/src/v2/methods/legalRegulatory.tsnew file mode 100644index 000000000..eab152a22--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/legalRegulatory.ts@@ -0,0 +1,37 @@+import type {+ LegalRegulatorySearchOptions,+ LegalRegulatorySearchResponse,+} from "../types";+import { HttpClient } from "../utils/httpClient";+import {+ normalizeAxiosError,+ throwForBadResponse,+} from "../utils/errorHandler";++const ENDPOINT = "/v2/search/legal-regulatory";++/** Search the Legal and Regulatory Index. */+export async function legalRegulatorySearch(+ http: HttpClient,+ query: string,+ options: LegalRegulatorySearchOptions = {},+): Promise<LegalRegulatorySearchResponse> {+ if (!query || !query.trim()) throw new Error("query cannot be empty");++ try {+ const response = await http.post<LegalRegulatorySearchResponse>(ENDPOINT, {+ query,+ ...(options.k !== undefined ? { k: options.k } : {}),+ });++ if (response.status !== 200 || !response.data?.success) {+ throwForBadResponse(response, "search legal and regulatory sources");+ }+ return response.data;+ } catch (error: any) {+ if (error?.isAxiosError) {+ return normalizeAxiosError(error, "search legal and regulatory sources");+ }+ throw error;+ }+}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex f82e0c6e1..8af462474 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -806,6 +806,16 @@ export interface DeveloperSearchResponse {sources?: DeveloperSearchSourceStatus[];}+export interface LegalRegulatorySearchOptions {+ /** Total ranked results, 1–100 (default 10). */+ k?: number;+}++export interface LegalRegulatorySearchResponse {+ success: boolean;+ data: { web: SearchResultWeb[] };+}+export interface SearchResultWeb {url: string;title?: string;diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex d6ded65d2..c8b8c994a 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -258,6 +258,20 @@ print(evidence.repos) # indexed-status echoes for requested repos`archived`, `fork`, and `skills="only"`. Supplying both `repos` and `sources`OR-combines GitHub-backed and documentation results.+### Legal and regulatory search++Use `legal_regulatory_search` to search the Legal and Regulatory Index: primary+law and regulatory material from US federal, state, and local government+sources, including statutes, regulations, codes, court opinions, and other+government publications. Results come back in the ordinary web-result shape.++```python+law = firecrawl.legal_regulatory_search("food labeling requirements", k=5)++for result in law.data.web:+ print(result.position, result.title, result.url)+```+### Research / paper searchUse `search_papers` to search Firecrawl's research paper index: ~43M paperdiff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_legal_regulatory_search.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_legal_regulatory_search.pynew file mode 100644index 000000000..44026e2da--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_legal_regulatory_search.py@@ -0,0 +1,81 @@+from unittest.mock import AsyncMock, Mock++import pytest++from firecrawl.v2.client import FirecrawlClient+from firecrawl.v2.client_async import AsyncFirecrawlClient+++RESPONSE = {+ "success": True,+ "data": {+ "web": [+ {+ "url": "https://www.ecfr.gov/current/title-21/chapter-I/subchapter-B/part-101",+ "title": "21 CFR Part 101 -- Food Labeling",+ "description": "matched snippet",+ "position": 1,+ }+ ]+ },+}+++def _response():+ response = Mock()+ response.status_code = 200+ response.json.return_value = RESPONSE+ return response+++def test_legal_regulatory_search_posts_query_and_k_and_parses_web_results():+ transport = Mock()+ transport.post.return_value = _response()+ client = FirecrawlClient.__new__(FirecrawlClient)+ client.http_client = transport++ result = client.legal_regulatory_search("food labeling requirements", k=5)++ transport.post.assert_called_once_with(+ "/v2/search/legal-regulatory", {"query": "food labeling requirements", "k": 5}+ )+ assert result.data.web[0].title == "21 CFR Part 101 -- Food Labeling"+ assert result.data.web[0].position == 1+++def test_legal_regulatory_search_omits_k_when_not_provided():+ transport = Mock()+ transport.post.return_value = _response()+ client = FirecrawlClient.__new__(FirecrawlClient)+ client.http_client = transport++ client.legal_regulatory_search("zoning variance")++ transport.post.assert_called_once_with(+ "/v2/search/legal-regulatory", {"query": "zoning variance"}+ )+++def test_legal_regulatory_search_rejects_empty_query():+ transport = Mock()+ client = FirecrawlClient.__new__(FirecrawlClient)+ client.http_client = transport++ with pytest.raises(ValueError, match="query cannot be empty"):+ client.legal_regulatory_search(" ")+ transport.post.assert_not_called()+++async def test_async_legal_regulatory_search_posts_query_and_k():+ transport = Mock()+ transport.post = AsyncMock(return_value=_response())+ client = AsyncFirecrawlClient.__new__(AsyncFirecrawlClient)+ client.async_http_client = transport++ result = await client.legal_regulatory_search("food labeling requirements", k=5)++ transport.post.assert_awaited_once_with(+ "/v2/search/legal-regulatory", {"query": "food labeling requirements", "k": 5}+ )+ assert result.data.web[0].url.startswith("https://www.ecfr.gov/")diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex 1a3338d47..3919c063b 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -82,6 +82,7 @@ class V2Proxy:self.get_parse_formats = client_instance.get_parse_formatsself.search = client_instance.searchself.developer_search = client_instance.developer_search+ self.legal_regulatory_search = client_instance.legal_regulatory_searchself.crawl = client_instance.crawlself.start_crawl = client_instance.start_crawlself.get_crawl_status = client_instance.get_crawl_status@@ -175,6 +176,7 @@ class AsyncV2Proxy:self.get_parse_formats = client_instance.get_parse_formatsself.search = client_instance.searchself.developer_search = client_instance.developer_search+ self.legal_regulatory_search = client_instance.legal_regulatory_searchself.crawl = client_instance.crawlself.start_crawl = client_instance.start_crawlself.wait_crawl = client_instance.wait_crawl@@ -292,6 +294,7 @@ class Firecrawl:self.get_parse_formats = self._v2_client.get_parse_formatsself.search = self._v2_client.searchself.developer_search = self._v2_client.developer_search+ self.legal_regulatory_search = self._v2_client.legal_regulatory_searchself.map = self._v2_client.mapself.create_monitor = self._v2_client.create_monitorself.list_monitors = self._v2_client.list_monitors@@ -437,6 +440,7 @@ class AsyncFirecrawl:self.get_parse_formats = self._v2_client.get_parse_formatsself.search = self._v2_client.searchself.developer_search = self._v2_client.developer_search+ self.legal_regulatory_search = self._v2_client.legal_regulatory_searchself.map = self._v2_client.mapself.create_monitor = self._v2_client.create_monitorself.list_monitors = self._v2_client.list_monitorsdiff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.pyindex 34b635193..ed127f24b 100644--- a/apps/python-sdk/firecrawl/types.py+++ b/apps/python-sdk/firecrawl/types.py@@ -69,6 +69,9 @@ from .v2.types import (DeveloperSearchRepoStatus,DeveloperSearchSourceStatus,DeveloperSearchResponse,+ LegalRegulatorySearchRequest,+ LegalRegulatorySearchData,+ LegalRegulatorySearchResponse,# Action typesWaitAction,@@ -169,6 +172,9 @@ __all__ = ['DeveloperSearchRepoStatus','DeveloperSearchSourceStatus','DeveloperSearchResponse',+ 'LegalRegulatorySearchRequest',+ 'LegalRegulatorySearchData',+ 'LegalRegulatorySearchResponse',# Action types'WaitAction',diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 8a6cf6994..3aaea5a0f 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -17,6 +17,7 @@ from .types import (SearchData,DeveloperSearchResponse,DeveloperSearchType,+ LegalRegulatorySearchResponse,SourceOption,CategoryOption,FindToolsData,@@ -69,6 +70,7 @@ from .methods import crawl as crawl_modulefrom .methods import batch as batch_modulefrom .methods import search as search_modulefrom .methods import developer as developer_module+from .methods import legal_regulatory as legal_regulatory_modulefrom .methods import map as map_modulefrom .methods import batch as batch_methodsfrom .methods import usage as usage_methods@@ -584,6 +586,16 @@ class FirecrawlClient:fork=fork,skills=skills,)++ def legal_regulatory_search(+ self,+ query: str,+ k: Optional[int] = None,+ ) -> LegalRegulatorySearchResponse:+ """Search the Legal and Regulatory Index of US primary law and regulatory material."""+ return legal_regulatory_module.legal_regulatory_search(+ self.http_client, query, k=k+ )def crawl(self,diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 4d3f552be..c698d9569 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -20,6 +20,7 @@ from .types import (SearchData,DeveloperSearchResponse,DeveloperSearchType,+ LegalRegulatorySearchResponse,SourceOption,FindToolsData,AlexandriaCall,@@ -65,6 +66,7 @@ from .methods.aio import batch as async_batch # type: ignore[attr-defined]from .methods.aio import crawl as async_crawl # type: ignore[attr-defined]from .methods.aio import search as async_search # type: ignore[attr-defined]from .methods.aio import developer as async_developer # type: ignore[attr-defined]+from .methods.aio import legal_regulatory as async_legal_regulatory # type: ignore[attr-defined]from .methods.aio import map as async_map # type: ignore[attr-defined]from .methods.aio import usage as async_usage # type: ignore[attr-defined]from .methods.aio import extract as async_extract # type: ignore[attr-defined]@@ -356,6 +358,16 @@ class AsyncFirecrawlClient:skills=skills,)+ async def legal_regulatory_search(+ self,+ query: str,+ k: Optional[int] = None,+ ) -> LegalRegulatorySearchResponse:+ """Search the Legal and Regulatory Index of US primary law and regulatory material."""+ return await async_legal_regulatory.legal_regulatory_search(+ self.async_http_client, query, k=k+ )+async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:if kwargs.get("scrape_options") is None:scrape_kwargs = {k: kwargs.pop(k) for k in list(kwargs) if k in _SCRAPE_OPTION_KEYS and kwargs[k] is not None}diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/legal_regulatory.py b/apps/python-sdk/firecrawl/v2/methods/aio/legal_regulatory.pynew file mode 100644index 000000000..bf6d769f0--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/aio/legal_regulatory.py@@ -0,0 +1,27 @@+"""Async Legal and Regulatory Index search functionality for Firecrawl v2."""++from typing import Optional++from ...types import LegalRegulatorySearchRequest, LegalRegulatorySearchResponse+from ...utils.error_handler import handle_response_error+from ...utils.http_client_async import AsyncHttpClient+++ENDPOINT = "/v2/search/legal-regulatory"+++async def legal_regulatory_search(+ client: AsyncHttpClient,+ query: str,+ *,+ k: Optional[int] = None,+) -> LegalRegulatorySearchResponse:+ """Search the Legal and Regulatory Index asynchronously."""+ if not query or not query.strip():+ raise ValueError("query cannot be empty")++ request = LegalRegulatorySearchRequest(query=query, k=k)+ response = await client.post(ENDPOINT, request.model_dump(exclude_none=True))+ if response.status_code != 200:+ handle_response_error(response, "search legal and regulatory sources")+ return LegalRegulatorySearchResponse.model_validate(response.json())diff --git a/apps/python-sdk/firecrawl/v2/methods/legal_regulatory.py b/apps/python-sdk/firecrawl/v2/methods/legal_regulatory.pynew file mode 100644index 000000000..16bacc0e6--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/legal_regulatory.py@@ -0,0 +1,26 @@+"""Legal and Regulatory Index search functionality for Firecrawl v2."""++from typing import Optional++from ..types import LegalRegulatorySearchRequest, LegalRegulatorySearchResponse+from ..utils import HttpClient, handle_response_error+++ENDPOINT = "/v2/search/legal-regulatory"+++def legal_regulatory_search(+ client: HttpClient,+ query: str,+ *,+ k: Optional[int] = None,+) -> LegalRegulatorySearchResponse:+ """Search the Legal and Regulatory Index."""+ if not query or not query.strip():+ raise ValueError("query cannot be empty")++ request = LegalRegulatorySearchRequest(query=query, k=k)+ response = client.post(ENDPOINT, request.model_dump(exclude_none=True))+ if response.status_code != 200:+ handle_response_error(response, "search legal and regulatory sources")+ return LegalRegulatorySearchResponse.model_validate(response.json())diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 04a06681a..3c0738b8f 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -2412,6 +2412,22 @@ class DeveloperSearchResponse(BaseModel):sources: Optional[List[DeveloperSearchSourceStatus]] = None+class LegalRegulatorySearchRequest(BaseModel):+ """Request for the Legal and Regulatory Index search endpoint."""++ query: str+ k: Optional[int] = Field(default=None, ge=1, le=100)+++class LegalRegulatorySearchData(BaseModel):+ web: List[SearchResultWeb]+++class LegalRegulatorySearchResponse(BaseModel):+ success: bool+ data: LegalRegulatorySearchData++class SearchRequest(BaseModel):"""Request for search operations."""