Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
python-sdk: add dedicated developer search
Karan Lokchandani643964fef74f
diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex ad571ee4d..91d9d4e4e 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -211,6 +211,35 @@ results = firecrawl.search("nanopore basecalling accuracy", categories=["researc> pubmed.ncbi.nlm.nih.gov, nature.com, biorxiv.org, ...) and returns web page> results. To search papers themselves, use `search_papers` below.+### Developer search++Use `developer_search` for the dedicated developer index and its complete+filter and response contract. Generic `search(categories=["developer"])`+returns only the first passage as a description and does not accept these+filters.++```python+evidence = firecrawl.developer_search(+ "configure retry backoff",+ repos=["firecrawl/firecrawl"],+ types=["issue", "pull_request", "readme"],+ passages=3,+ language="TypeScript",+ license="MIT",+)++for result in evidence.results:+ # Result kind is the id prefix; the API intentionally omits a type field.+ print(result.id, result.license)+ for passage in result.passages:+ print(passage.text, passage.citation_url)+print(evidence.repos) # indexed-status echoes for requested repos+```++`developer_search` also supports `sources`, `topic`, `min_stars`, `max_stars`,+`archived`, `fork`, and `skills="only"`. Supplying both `repos` and `sources`+OR-combines GitHub-backed and documentation results.+### Research / paper searchUse `search_papers` to search Firecrawl's research paper index: ~43M paperdiff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_developer_search.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_developer_search.pynew file mode 100644index 000000000..e459be40e--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_developer_search.py@@ -0,0 +1,105 @@+from unittest.mock import AsyncMock, Mock++import pytest++from firecrawl.v2.client import FirecrawlClient+from firecrawl.v2.client_async import AsyncFirecrawlClient+from firecrawl.v2.types import DeveloperSearchLicenseDisclosure+++RESPONSE = {+ "success": True,+ "results": [+ {+ "id": "issue:firecrawl/firecrawl#1",+ "url": "https://github.com/firecrawl/firecrawl/issues/1",+ "passages": [+ {+ "text": "evidence",+ "citation_url": "https://github.com/firecrawl/firecrawl/issues/1#issuecomment-1",+ }+ ],+ "license": {"state": "licensed", "spdx_id": "MIT"},+ }+ ],+ "repos": [+ {+ "repo": "firecrawl/firecrawl",+ "indexed": True,+ "types": {"issue": True, "pullRequest": True, "readme": True},+ }+ ],+ "sources": [{"source": "firecrawl", "indexed": False}],+}+++def _response(payload=RESPONSE):+ response = Mock()+ response.status_code = 200+ response.json.return_value = payload+ return response+++def _options():+ return {+ "k": 20,+ "passages": 5,+ "types": ["doc", "issue", "pull_request", "readme"],+ "repos": ["firecrawl/firecrawl"],+ "sources": ["firecrawl"],+ "language": "TypeScript",+ "topic": ["web-scraping"],+ "license": "MIT",+ "min_stars": 100,+ "max_stars": 50000,+ "archived": False,+ "fork": True,+ "skills": "only",+ }+++EXPECTED_REQUEST = {"query": "retry configuration", **_options()}+++def test_developer_search_forwards_all_filters_and_parses_evidence():+ transport = Mock()+ transport.post.return_value = _response()+ client = FirecrawlClient.__new__(FirecrawlClient)+ client.http_client = transport++ result = client.developer_search("retry configuration", **_options())++ transport.post.assert_called_once_with("/v2/search/developer", EXPECTED_REQUEST)+ assert result.results[0].passages[0].citation_url.endswith("issuecomment-1")+ assert isinstance(result.results[0].license, DeveloperSearchLicenseDisclosure)+ assert result.results[0].license.spdx_id == "MIT"+ assert result.repos[0].types.pull_request is True+ assert result.sources[0].indexed is False+++def test_developer_search_accepts_flattened_spdx_license():+ payload = {+ **RESPONSE,+ "results": [{**RESPONSE["results"][0], "license": "MIT"}],+ }+ transport = Mock()+ transport.post.return_value = _response(payload)+ client = FirecrawlClient.__new__(FirecrawlClient)+ client.http_client = transport++ result = client.developer_search("license shape")++ assert result.results[0].license == "MIT"+++async def test_async_developer_search_forwards_all_filters():+ transport = Mock()+ transport.post = AsyncMock(return_value=_response())+ client = AsyncFirecrawlClient.__new__(AsyncFirecrawlClient)+ client.async_http_client = transport++ result = await client.developer_search("retry configuration", **_options())++ transport.post.assert_awaited_once_with("/v2/search/developer", EXPECTED_REQUEST)+ assert result.results[0].id == "issue:firecrawl/firecrawl#1"diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex eb148aed9..b648b4bef 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -78,6 +78,7 @@ class V2Proxy:self.delete_scrape_browser = self.stop_interactionself.parse = client_instance.parseself.search = client_instance.search+ self.developer_search = client_instance.developer_searchself.crawl = client_instance.crawlself.start_crawl = client_instance.start_crawlself.get_crawl_status = client_instance.get_crawl_status@@ -164,6 +165,7 @@ class AsyncV2Proxy:self.delete_scrape_browser = self.stop_interactionself.parse = client_instance.parseself.search = client_instance.search+ self.developer_search = client_instance.developer_searchself.crawl = client_instance.crawlself.start_crawl = client_instance.start_crawlself.wait_crawl = client_instance.wait_crawl@@ -270,6 +272,7 @@ class Firecrawl:self.delete_scrape_browser = self.stop_interactionself.parse = self._v2_client.parseself.search = self._v2_client.search+ self.developer_search = self._v2_client.developer_searchself.map = self._v2_client.mapself.create_monitor = self._v2_client.create_monitorself.list_monitors = self._v2_client.list_monitors@@ -405,6 +408,7 @@ class AsyncFirecrawl:self.delete_scrape_browser = self.stop_interactionself.parse = self._v2_client.parseself.search = self._v2_client.search+ self.developer_search = self._v2_client.developer_searchself.map = self._v2_client.mapself.create_monitor = self._v2_client.create_monitorself.list_monitors = self._v2_client.list_monitorsdiff --git a/apps/python-sdk/firecrawl/types.py b/apps/python-sdk/firecrawl/types.pyindex 47dea6159..956e4ce2b 100644--- a/apps/python-sdk/firecrawl/types.py+++ b/apps/python-sdk/firecrawl/types.py@@ -60,6 +60,15 @@ from .v2.types import (SearchResultNews,SearchResultImages,SearchData,+ DeveloperSearchType,+ DeveloperSearchRequest,+ DeveloperSearchLicenseDisclosure,+ DeveloperSearchPassage,+ DeveloperSearchResult,+ DeveloperSearchRepoTypes,+ DeveloperSearchRepoStatus,+ DeveloperSearchSourceStatus,+ DeveloperSearchResponse,# Action typesWaitAction,@@ -150,6 +159,15 @@ __all__ = ['SearchResultNews','SearchResultImages','SearchData',+ 'DeveloperSearchType',+ 'DeveloperSearchRequest',+ 'DeveloperSearchLicenseDisclosure',+ 'DeveloperSearchPassage',+ 'DeveloperSearchResult',+ 'DeveloperSearchRepoTypes',+ 'DeveloperSearchRepoStatus',+ 'DeveloperSearchSourceStatus',+ 'DeveloperSearchResponse',# Action types'WaitAction',diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 5f65ca8a8..8c8f1ccbf 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -14,6 +14,8 @@ from .types import (Document,SearchRequest,SearchData,+ DeveloperSearchResponse,+ DeveloperSearchType,SourceOption,CategoryOption,CrawlRequest,@@ -60,6 +62,7 @@ from .methods import parse as parse_modulefrom .methods import crawl as crawl_modulefrom .methods import batch as batch_modulefrom .methods import search as search_module+from .methods import developer as developer_modulefrom .methods import map as map_modulefrom .methods import batch as batch_methodsfrom .methods import usage as usage_methods@@ -459,6 +462,43 @@ class FirecrawlClient:)return search_module.search(self.http_client, request)++ def developer_search(+ self,+ query: str,+ *,+ k: Optional[int] = None,+ passages: Optional[int] = None,+ types: Optional[List[DeveloperSearchType]] = None,+ repos: Optional[List[str]] = None,+ sources: Optional[List[str]] = None,+ language: Optional[str] = None,+ topic: Optional[List[str]] = None,+ license: Optional[str] = None,+ min_stars: Optional[int] = None,+ max_stars: Optional[int] = None,+ archived: Optional[bool] = None,+ fork: Optional[bool] = None,+ skills: Optional[Literal["only"]] = None,+ ) -> DeveloperSearchResponse:+ """Search the dedicated developer index with full filters and evidence."""+ return developer_module.developer_search(+ self.http_client,+ query,+ k=k,+ passages=passages,+ types=types,+ repos=repos,+ sources=sources,+ language=language,+ topic=topic,+ license=license,+ min_stars=min_stars,+ max_stars=max_stars,+ archived=archived,+ fork=fork,+ skills=skills,+ )def crawl(self,diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 587c823fc..42b901bc3 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -16,6 +16,8 @@ from .types import (MonitorWebhookConfig,SearchRequest,SearchData,+ DeveloperSearchResponse,+ DeveloperSearchType,SourceOption,CrawlResponse,CrawlJob,@@ -56,6 +58,7 @@ from .methods.aio import parse as async_parse # type: ignore[attr-defined]from .methods.aio import batch as async_batch # type: ignore[attr-defined]from .methods.aio import crawl as async_crawl # type: ignore[attr-defined]from .methods.aio import search as async_search # type: ignore[attr-defined]+from .methods.aio import developer as async_developer # type: ignore[attr-defined]from .methods.aio import map as async_map # type: ignore[attr-defined]from .methods.aio import usage as async_usage # type: ignore[attr-defined]from .methods.aio import extract as async_extract # type: ignore[attr-defined]@@ -257,6 +260,43 @@ class AsyncFirecrawlClient:request = SearchRequest(query=query, **{k: v for k, v in kwargs.items() if v is not None})return await async_search.search(self.async_http_client, request)+ async def developer_search(+ self,+ query: str,+ *,+ k: Optional[int] = None,+ passages: Optional[int] = None,+ types: Optional[List[DeveloperSearchType]] = None,+ repos: Optional[List[str]] = None,+ sources: Optional[List[str]] = None,+ language: Optional[str] = None,+ topic: Optional[List[str]] = None,+ license: Optional[str] = None,+ min_stars: Optional[int] = None,+ max_stars: Optional[int] = None,+ archived: Optional[bool] = None,+ fork: Optional[bool] = None,+ skills: Optional[Literal["only"]] = None,+ ) -> DeveloperSearchResponse:+ """Search the dedicated developer index with full filters and evidence."""+ return await async_developer.developer_search(+ self.async_http_client,+ query,+ k=k,+ passages=passages,+ types=types,+ repos=repos,+ sources=sources,+ language=language,+ topic=topic,+ license=license,+ min_stars=min_stars,+ max_stars=max_stars,+ archived=archived,+ fork=fork,+ skills=skills,+ )+async def start_crawl(self, url: str, **kwargs) -> CrawlResponse:if kwargs.get("scrape_options") is None:scrape_kwargs = {k: kwargs.pop(k) for k in list(kwargs) if k in _SCRAPE_OPTION_KEYS and kwargs[k] is not None}diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/developer.py b/apps/python-sdk/firecrawl/v2/methods/aio/developer.pynew file mode 100644index 000000000..c2a4aeddc--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/aio/developer.py@@ -0,0 +1,58 @@+"""Async dedicated developer-search functionality for Firecrawl v2."""++from typing import List, Literal, Optional++from ...types import (+ DeveloperSearchRequest,+ DeveloperSearchResponse,+ DeveloperSearchType,+)+from ...utils.error_handler import handle_response_error+from ...utils.http_client_async import AsyncHttpClient+++ENDPOINT = "/v2/search/developer"+++async def developer_search(+ client: AsyncHttpClient,+ query: str,+ *,+ k: Optional[int] = None,+ passages: Optional[int] = None,+ types: Optional[List[DeveloperSearchType]] = None,+ repos: Optional[List[str]] = None,+ sources: Optional[List[str]] = None,+ language: Optional[str] = None,+ topic: Optional[List[str]] = None,+ license: Optional[str] = None,+ min_stars: Optional[int] = None,+ max_stars: Optional[int] = None,+ archived: Optional[bool] = None,+ fork: Optional[bool] = None,+ skills: Optional[Literal["only"]] = None,+) -> DeveloperSearchResponse:+ """Search the developer index asynchronously with all dedicated filters."""+ if not query or not query.strip():+ raise ValueError("query cannot be empty")++ request = DeveloperSearchRequest(+ query=query,+ k=k,+ passages=passages,+ types=types,+ repos=repos,+ sources=sources,+ language=language,+ topic=topic,+ license=license,+ min_stars=min_stars,+ max_stars=max_stars,+ archived=archived,+ fork=fork,+ skills=skills,+ )+ response = await client.post(ENDPOINT, request.model_dump(exclude_none=True))+ if response.status_code != 200:+ handle_response_error(response, "search developer sources")+ return DeveloperSearchResponse.model_validate(response.json())diff --git a/apps/python-sdk/firecrawl/v2/methods/developer.py b/apps/python-sdk/firecrawl/v2/methods/developer.pynew file mode 100644index 000000000..974d38db9--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/developer.py@@ -0,0 +1,57 @@+"""Dedicated developer-search functionality for Firecrawl v2."""++from typing import List, Literal, Optional++from ..types import (+ DeveloperSearchRequest,+ DeveloperSearchResponse,+ DeveloperSearchType,+)+from ..utils import HttpClient, handle_response_error+++ENDPOINT = "/v2/search/developer"+++def developer_search(+ client: HttpClient,+ query: str,+ *,+ k: Optional[int] = None,+ passages: Optional[int] = None,+ types: Optional[List[DeveloperSearchType]] = None,+ repos: Optional[List[str]] = None,+ sources: Optional[List[str]] = None,+ language: Optional[str] = None,+ topic: Optional[List[str]] = None,+ license: Optional[str] = None,+ min_stars: Optional[int] = None,+ max_stars: Optional[int] = None,+ archived: Optional[bool] = None,+ fork: Optional[bool] = None,+ skills: Optional[Literal["only"]] = None,+) -> DeveloperSearchResponse:+ """Search the developer index with its complete filter surface."""+ if not query or not query.strip():+ raise ValueError("query cannot be empty")++ request = DeveloperSearchRequest(+ query=query,+ k=k,+ passages=passages,+ types=types,+ repos=repos,+ sources=sources,+ language=language,+ topic=topic,+ license=license,+ min_stars=min_stars,+ max_stars=max_stars,+ archived=archived,+ fork=fork,+ skills=skills,+ )+ response = client.post(ENDPOINT, request.model_dump(exclude_none=True))+ if response.status_code != 200:+ handle_response_error(response, "search developer sources")+ return DeveloperSearchResponse.model_validate(response.json())diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex fc5199e95..1fe6bf4bc 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -1695,6 +1695,75 @@ class Location(BaseModel):languages: Optional[List[str]] = None+DeveloperSearchType = Literal["doc", "issue", "pull_request", "readme"]+++class DeveloperSearchRequest(BaseModel):+ """Request for the dedicated developer-search endpoint."""++ query: str+ k: Optional[int] = Field(default=None, ge=1, le=100)+ passages: Optional[int] = Field(default=None, ge=1, le=5)+ types: Optional[List[DeveloperSearchType]] = Field(default=None, max_length=4)+ repos: Optional[List[str]] = Field(default=None, max_length=20)+ sources: Optional[List[str]] = Field(default=None, max_length=20)+ language: Optional[str] = None+ topic: Optional[List[str]] = Field(default=None, max_length=8)+ license: Optional[str] = None+ min_stars: Optional[int] = Field(default=None, ge=0)+ max_stars: Optional[int] = Field(default=None, ge=0)+ archived: Optional[bool] = None+ fork: Optional[bool] = None+ skills: Optional[Literal["only"]] = None+++class DeveloperSearchLicenseDisclosure(BaseModel):+ """Repository license disclosure returned by developer search."""++ state: Literal["licensed", "known_absent", "unknown"]+ spdx_id: Optional[str] = None+++class DeveloperSearchPassage(BaseModel):+ text: str+ citation_url: Optional[str] = None+++class DeveloperSearchResult(BaseModel):+ id: str+ url: str+ title: Optional[str] = None+ passages: List[DeveloperSearchPassage]+ # Accept both shapes while the API flattens license objects to SPDX strings.+ license: Optional[Union[DeveloperSearchLicenseDisclosure, str]] = None+++class DeveloperSearchRepoTypes(BaseModel):+ model_config = {"populate_by_name": True}++ issue: bool+ pull_request: bool = Field(alias="pullRequest")+ readme: bool+++class DeveloperSearchRepoStatus(BaseModel):+ repo: str+ indexed: bool+ types: DeveloperSearchRepoTypes+++class DeveloperSearchSourceStatus(BaseModel):+ source: str+ indexed: bool+++class DeveloperSearchResponse(BaseModel):+ success: bool+ results: List[DeveloperSearchResult]+ repos: Optional[List[DeveloperSearchRepoStatus]] = None+ sources: Optional[List[DeveloperSearchSourceStatus]] = None++class SearchRequest(BaseModel):"""Request for search operations."""