Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(sdks): add the developer search category

Karan Lokchandani6c1e92e25b99
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index 653253e4c..b67b98ab5 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,6 +1,6 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.31.1",
+ "version": "4.32.0",
"description": "JavaScript SDK for Firecrawl API",
"main": "dist/index.js",
"types": "dist/index.d.ts",
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts
new file mode 100644
index 000000000..43c3977bd
--- /dev/null
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts
@@ -0,0 +1,65 @@
+import { describe, expect, jest, test } from "@jest/globals";
+import { search } from "../../../v2/methods/search";
+
+const developerResult = {
+ title: "https://github.com/firecrawl/firecrawl/issues/1",
+ url: "https://github.com/firecrawl/firecrawl/issues/1",
+ description: "passage",
+ position: 1,
+ category: "developer",
+};
+
+function httpWith(data: Record<string, unknown>) {
+ return {
+ post: jest.fn(async () => ({ status: 200, data: { success: true, data } })),
+ } as any;
+}
+
+describe("v2 search developer category", () => {
+ test("forwards the developer category", async () => {
+ const http = httpWith({});
+
+ await search(http, { query: "firecrawl", categories: ["developer"] });
+
+ expect(http.post).toHaveBeenCalledWith(
+ "/v2/search",
+ { query: "firecrawl", categories: ["developer"] },
+ {},
+ );
+ });
+
+ test("returns developer results", async () => {
+ const http = httpWith({
+ web: [{ title: "W", url: "http://a.com" }],
+ developer: [developerResult],
+ });
+
+ const result = await search(http, {
+ query: "firecrawl",
+ categories: [{ type: "developer" }],
+ });
+
+ expect(result.web).toHaveLength(1);
+ expect(result.developer).toHaveLength(1);
+ expect(result.developer?.[0]).toEqual(developerResult);
+ });
+
+ test("omits developer when the response has none", async () => {
+ const http = httpWith({ web: [] });
+
+ const result = await search(http, { query: "firecrawl" });
+
+ expect(result.developer).toBeUndefined();
+ });
+
+ test("lists developer in the .data error", async () => {
+ const http = httpWith({ developer: [developerResult] });
+
+ const result = await search(http, {
+ query: "firecrawl",
+ categories: ["developer"],
+ });
+
+ expect(() => (result as any).data).toThrow(/\.developer \(1 results\)/);
+ });
+});
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
index 00332394e..89b6506b9 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
@@ -102,12 +102,16 @@ export async function search(
if (data.news) out.news = transformArray<SearchResultNews>(data.news);
if (data.images)
out.images = transformArray<SearchResultImages>(data.images);
+ if (data.developer)
+ out.developer = transformArray<SearchResultWeb>(data.developer);
Object.defineProperty(out, "data", {
get() {
const parts: string[] = [];
if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
if (out.images?.length) parts.push(`.images (${out.images.length} results)`);
+ if (out.developer?.length)
+ parts.push(`.developer (${out.developer.length} results)`);
const available = parts.length ? parts.join(", ") : ".web, .news, or .images";
throw new Error(
`SearchData has no '.data'. Results are grouped by source: ${available}`,
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index c7ebc8eed..d2bb9bae3 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -683,10 +683,11 @@ export interface SearchData {
web?: Array<SearchResultWeb | Document>;
news?: Array<SearchResultNews | Document>;
images?: Array<SearchResultImages | Document>;
+ developer?: Array<SearchResultWeb | Document>;
}
export interface CategoryOption {
- type: "github" | "research" | "pdf";
+ type: "github" | "research" | "pdf" | "developer";
}
export interface SearchRequest {
@@ -694,7 +695,9 @@ export interface SearchRequest {
sources?: Array<
"web" | "news" | "images" | { type: "web" | "news" | "images" }
>;
- categories?: Array<"github" | "research" | "pdf" | CategoryOption>;
+ categories?: Array<
+ "github" | "research" | "pdf" | "developer" | CategoryOption
+ >;
includeDomains?: string[];
excludeDomains?: string[];
limit?: number;
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 7a1a6e623..02bad5b9d 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -18,7 +18,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.33.1"
+__version__ = "4.34.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py
new file mode 100644
index 000000000..bfaaf253b
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py
@@ -0,0 +1,72 @@
+import pytest
+from unittest.mock import AsyncMock, Mock
+
+from firecrawl.v2.types import SearchRequest, SearchResultWeb
+from firecrawl.v2.methods.search import search
+from firecrawl.v2.methods.aio.search import search as search_async
+
+
+def _response(data):
+ response = Mock()
+ response.status_code = 200
+ response.json.return_value = {"success": True, "data": data}
+ return response
+
+
+DEVELOPER_PAYLOAD = {
+ "web": [{"title": "W", "url": "http://a.com", "description": "d"}],
+ "developer": [
+ {
+ "title": "https://github.com/firecrawl/firecrawl/issues/1",
+ "url": "https://github.com/firecrawl/firecrawl/issues/1",
+ "description": "passage",
+ "position": 1,
+ "category": "developer",
+ }
+ ],
+}
+
+
+class TestSearchDeveloperResults:
+ """Unit tests for the developer category results."""
+
+ def test_developer_results_are_parsed(self):
+ client = Mock()
+ client.post.return_value = _response(DEVELOPER_PAYLOAD)
+
+ result = search(client, SearchRequest(query="test", categories=["developer"]))
+
+ assert len(result.web) == 1
+ assert len(result.developer) == 1
+ assert isinstance(result.developer[0], SearchResultWeb)
+ assert result.developer[0].url == "https://github.com/firecrawl/firecrawl/issues/1"
+ assert result.developer[0].category == "developer"
+
+ def test_developer_absent_when_not_returned(self):
+ client = Mock()
+ client.post.return_value = _response({"web": []})
+
+ result = search(client, SearchRequest(query="test"))
+
+ assert result.developer is None
+
+ def test_data_property_lists_developer(self):
+ client = Mock()
+ client.post.return_value = _response({"developer": DEVELOPER_PAYLOAD["developer"]})
+
+ result = search(client, SearchRequest(query="test", categories=["developer"]))
+
+ with pytest.raises(AttributeError, match=r"\.developer \(1 results\)"):
+ result.data
+
+ @pytest.mark.asyncio
+ async def test_developer_results_are_parsed_async(self):
+ client = Mock()
+ client.post = AsyncMock(return_value=_response(DEVELOPER_PAYLOAD))
+
+ result = await search_async(
+ client, SearchRequest(query="test", categories=["developer"])
+ )
+
+ assert len(result.developer) == 1
+ assert result.developer[0].url == "https://github.com/firecrawl/firecrawl/issues/1"
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py
index 406fdde49..7c293128a 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py
@@ -1,5 +1,5 @@
import pytest
-from firecrawl.v2.types import SearchRequest, Source, ScrapeOptions, ScrapeFormats
+from firecrawl.v2.types import Category, SearchRequest, Source, ScrapeOptions, ScrapeFormats
from firecrawl.v2.methods.search import _validate_search_request
@@ -67,6 +67,23 @@ class TestSearchValidation:
with pytest.raises(ValueError, match="Invalid source type"):
_validate_search_request(request)
+ def test_validate_categories(self):
+ """Test validation of search categories."""
+ for category in ["github", "research", "pdf", "developer"]:
+ request = SearchRequest(query="test", categories=[category])
+ assert _validate_search_request(request) == request
+
+ request = SearchRequest(query="test", categories=[Category(type=category)])
+ assert _validate_search_request(request) == request
+
+ request = SearchRequest(query="test", categories=["invalid_category"])
+ with pytest.raises(ValueError, match="Invalid category type"):
+ _validate_search_request(request)
+
+ request = SearchRequest(query="test", categories=[Category(type="invalid_category")])
+ with pytest.raises(ValueError, match="Invalid category type"):
+ _validate_search_request(request)
+
def test_validate_invalid_location(self):
"""Test validation of invalid location."""
# Empty location
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/search.py b/apps/python-sdk/firecrawl/v2/methods/aio/search.py
index 02f5e0bb0..6dbe2fd9b 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/search.py
@@ -47,6 +47,8 @@ async def search(
out.news = _transform_array(data["news"], SearchResultNews)
if "images" in data:
out.images = _transform_array(data["images"], SearchResultImages)
+ if "developer" in data:
+ out.developer = _transform_array(data["developer"], SearchResultWeb)
return out
except Exception as err:
if hasattr(err, "response"):
diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.py
index 7e6e30b0a..0f23ff00c 100644
--- a/apps/python-sdk/firecrawl/v2/methods/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/search.py
@@ -42,6 +42,8 @@ def search(
out.news = _transform_array(data["news"], SearchResultNews)
if "images" in data:
out.images = _transform_array(data["images"], SearchResultImages)
+ if "developer" in data:
+ out.developer = _transform_array(data["developer"], SearchResultWeb)
return out
except Exception as err:
# If the error is an HTTP error from requests, handle it
@@ -133,7 +135,7 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:
# Validate categories (if provided)
if request.categories is not None:
- valid_categories = {"github", "research", "pdf"}
+ valid_categories = {"github", "research", "pdf", "developer"}
for category in request.categories:
if isinstance(category, str):
if category not in valid_categories:
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 08e9d9ba0..a8d286e38 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -560,6 +560,8 @@ class Category(BaseModel):
- "github": Filter results to GitHub repositories
- "research": Filter results to research papers and academic sites
- "pdf": Filter results to PDF files (adds filetype:pdf to search)
+ - "developer": Add developer results (issues, pull requests, READMEs and
+ documentation) under `.developer`
"""
type: str
@@ -1692,6 +1694,7 @@ class SearchData(BaseModel):
web: Optional[List[Union[SearchResultWeb, Document]]] = None
news: Optional[List[Union[SearchResultNews, Document]]] = None
images: Optional[List[Union[SearchResultImages, Document]]] = None
+ developer: Optional[List[Union[SearchResultWeb, Document]]] = None
@property
def data(self):
@@ -1702,6 +1705,8 @@ class SearchData(BaseModel):
parts.append(f".news ({len(self.news)} results)")
if self.images:
parts.append(f".images ({len(self.images)} results)")
+ if self.developer:
+ parts.append(f".developer ({len(self.developer)} results)")
available = ", ".join(parts) if parts else ".web, .news, or .images"
raise AttributeError(
f"SearchData has no '.data'. Results are grouped by source: {available}"