Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(sdks): add the developer search category
Karan Lokchandani6c1e92e25b99
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex 653253e4c..b67b98ab5 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,6 +1,6 @@{"name": "@mendable/firecrawl-js",- "version": "4.31.1",+ "version": "4.32.0","description": "JavaScript SDK for Firecrawl API","main": "dist/index.js","types": "dist/index.d.ts",diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.tsnew file mode 100644index 000000000..43c3977bd--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts@@ -0,0 +1,65 @@+import { describe, expect, jest, test } from "@jest/globals";+import { search } from "../../../v2/methods/search";++const developerResult = {+ title: "https://github.com/firecrawl/firecrawl/issues/1",+ url: "https://github.com/firecrawl/firecrawl/issues/1",+ description: "passage",+ position: 1,+ category: "developer",+};++function httpWith(data: Record<string, unknown>) {+ return {+ post: jest.fn(async () => ({ status: 200, data: { success: true, data } })),+ } as any;+}++describe("v2 search developer category", () => {+ test("forwards the developer category", async () => {+ const http = httpWith({});++ await search(http, { query: "firecrawl", categories: ["developer"] });++ expect(http.post).toHaveBeenCalledWith(+ "/v2/search",+ { query: "firecrawl", categories: ["developer"] },+ {},+ );+ });++ test("returns developer results", async () => {+ const http = httpWith({+ web: [{ title: "W", url: "http://a.com" }],+ developer: [developerResult],+ });++ const result = await search(http, {+ query: "firecrawl",+ categories: [{ type: "developer" }],+ });++ expect(result.web).toHaveLength(1);+ expect(result.developer).toHaveLength(1);+ expect(result.developer?.[0]).toEqual(developerResult);+ });++ test("omits developer when the response has none", async () => {+ const http = httpWith({ web: [] });++ const result = await search(http, { query: "firecrawl" });++ expect(result.developer).toBeUndefined();+ });++ test("lists developer in the .data error", async () => {+ const http = httpWith({ developer: [developerResult] });++ const result = await search(http, {+ query: "firecrawl",+ categories: ["developer"],+ });++ expect(() => (result as any).data).toThrow(/\.developer \(1 results\)/);+ });+});diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.tsindex 00332394e..89b6506b9 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts@@ -102,12 +102,16 @@ export async function search(if (data.news) out.news = transformArray<SearchResultNews>(data.news);if (data.images)out.images = transformArray<SearchResultImages>(data.images);+ if (data.developer)+ out.developer = transformArray<SearchResultWeb>(data.developer);Object.defineProperty(out, "data", {get() {const parts: string[] = [];if (out.web?.length) parts.push(`.web (${out.web.length} results)`);if (out.news?.length) parts.push(`.news (${out.news.length} results)`);if (out.images?.length) parts.push(`.images (${out.images.length} results)`);+ if (out.developer?.length)+ parts.push(`.developer (${out.developer.length} results)`);const available = parts.length ? parts.join(", ") : ".web, .news, or .images";throw new Error(`SearchData has no '.data'. Results are grouped by source: ${available}`,diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex c7ebc8eed..d2bb9bae3 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -683,10 +683,11 @@ export interface SearchData {web?: Array<SearchResultWeb | Document>;news?: Array<SearchResultNews | Document>;images?: Array<SearchResultImages | Document>;+ developer?: Array<SearchResultWeb | Document>;}export interface CategoryOption {- type: "github" | "research" | "pdf";+ type: "github" | "research" | "pdf" | "developer";}export interface SearchRequest {@@ -694,7 +695,9 @@ export interface SearchRequest {sources?: Array<"web" | "news" | "images" | { type: "web" | "news" | "images" }>;- categories?: Array<"github" | "research" | "pdf" | CategoryOption>;+ categories?: Array<+ "github" | "research" | "pdf" | "developer" | CategoryOption+ >;includeDomains?: string[];excludeDomains?: string[];limit?: number;diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 7a1a6e623..02bad5b9d 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -18,7 +18,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.33.1"+__version__ = "4.34.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.pynew file mode 100644index 000000000..bfaaf253b--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py@@ -0,0 +1,72 @@+import pytest+from unittest.mock import AsyncMock, Mock++from firecrawl.v2.types import SearchRequest, SearchResultWeb+from firecrawl.v2.methods.search import search+from firecrawl.v2.methods.aio.search import search as search_async+++def _response(data):+ response = Mock()+ response.status_code = 200+ response.json.return_value = {"success": True, "data": data}+ return response+++DEVELOPER_PAYLOAD = {+ "web": [{"title": "W", "url": "http://a.com", "description": "d"}],+ "developer": [+ {+ "title": "https://github.com/firecrawl/firecrawl/issues/1",+ "url": "https://github.com/firecrawl/firecrawl/issues/1",+ "description": "passage",+ "position": 1,+ "category": "developer",+ }+ ],+}+++class TestSearchDeveloperResults:+ """Unit tests for the developer category results."""++ def test_developer_results_are_parsed(self):+ client = Mock()+ client.post.return_value = _response(DEVELOPER_PAYLOAD)++ result = search(client, SearchRequest(query="test", categories=["developer"]))++ assert len(result.web) == 1+ assert len(result.developer) == 1+ assert isinstance(result.developer[0], SearchResultWeb)+ assert result.developer[0].url == "https://github.com/firecrawl/firecrawl/issues/1"+ assert result.developer[0].category == "developer"++ def test_developer_absent_when_not_returned(self):+ client = Mock()+ client.post.return_value = _response({"web": []})++ result = search(client, SearchRequest(query="test"))++ assert result.developer is None++ def test_data_property_lists_developer(self):+ client = Mock()+ client.post.return_value = _response({"developer": DEVELOPER_PAYLOAD["developer"]})++ result = search(client, SearchRequest(query="test", categories=["developer"]))++ with pytest.raises(AttributeError, match=r"\.developer \(1 results\)"):+ result.data++ @pytest.mark.asyncio+ async def test_developer_results_are_parsed_async(self):+ client = Mock()+ client.post = AsyncMock(return_value=_response(DEVELOPER_PAYLOAD))++ result = await search_async(+ client, SearchRequest(query="test", categories=["developer"])+ )++ assert len(result.developer) == 1+ assert result.developer[0].url == "https://github.com/firecrawl/firecrawl/issues/1"diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.pyindex 406fdde49..7c293128a 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_validation.py@@ -1,5 +1,5 @@import pytest-from firecrawl.v2.types import SearchRequest, Source, ScrapeOptions, ScrapeFormats+from firecrawl.v2.types import Category, SearchRequest, Source, ScrapeOptions, ScrapeFormatsfrom firecrawl.v2.methods.search import _validate_search_request@@ -67,6 +67,23 @@ class TestSearchValidation:with pytest.raises(ValueError, match="Invalid source type"):_validate_search_request(request)+ def test_validate_categories(self):+ """Test validation of search categories."""+ for category in ["github", "research", "pdf", "developer"]:+ request = SearchRequest(query="test", categories=[category])+ assert _validate_search_request(request) == request++ request = SearchRequest(query="test", categories=[Category(type=category)])+ assert _validate_search_request(request) == request++ request = SearchRequest(query="test", categories=["invalid_category"])+ with pytest.raises(ValueError, match="Invalid category type"):+ _validate_search_request(request)++ request = SearchRequest(query="test", categories=[Category(type="invalid_category")])+ with pytest.raises(ValueError, match="Invalid category type"):+ _validate_search_request(request)+def test_validate_invalid_location(self):"""Test validation of invalid location."""# Empty locationdiff --git a/apps/python-sdk/firecrawl/v2/methods/aio/search.py b/apps/python-sdk/firecrawl/v2/methods/aio/search.pyindex 02f5e0bb0..6dbe2fd9b 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/search.py@@ -47,6 +47,8 @@ async def search(out.news = _transform_array(data["news"], SearchResultNews)if "images" in data:out.images = _transform_array(data["images"], SearchResultImages)+ if "developer" in data:+ out.developer = _transform_array(data["developer"], SearchResultWeb)return outexcept Exception as err:if hasattr(err, "response"):diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.pyindex 7e6e30b0a..0f23ff00c 100644--- a/apps/python-sdk/firecrawl/v2/methods/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/search.py@@ -42,6 +42,8 @@ def search(out.news = _transform_array(data["news"], SearchResultNews)if "images" in data:out.images = _transform_array(data["images"], SearchResultImages)+ if "developer" in data:+ out.developer = _transform_array(data["developer"], SearchResultWeb)return outexcept Exception as err:# If the error is an HTTP error from requests, handle it@@ -133,7 +135,7 @@ def _validate_search_request(request: SearchRequest) -> SearchRequest:# Validate categories (if provided)if request.categories is not None:- valid_categories = {"github", "research", "pdf"}+ valid_categories = {"github", "research", "pdf", "developer"}for category in request.categories:if isinstance(category, str):if category not in valid_categories:diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 08e9d9ba0..a8d286e38 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -560,6 +560,8 @@ class Category(BaseModel):- "github": Filter results to GitHub repositories- "research": Filter results to research papers and academic sites- "pdf": Filter results to PDF files (adds filetype:pdf to search)+ - "developer": Add developer results (issues, pull requests, READMEs and+ documentation) under `.developer`"""type: str@@ -1692,6 +1694,7 @@ class SearchData(BaseModel):web: Optional[List[Union[SearchResultWeb, Document]]] = Nonenews: Optional[List[Union[SearchResultNews, Document]]] = Noneimages: Optional[List[Union[SearchResultImages, Document]]] = None+ developer: Optional[List[Union[SearchResultWeb, Document]]] = None@propertydef data(self):@@ -1702,6 +1705,8 @@ class SearchData(BaseModel):parts.append(f".news ({len(self.news)} results)")if self.images:parts.append(f".images ({len(self.images)} results)")+ if self.developer:+ parts.append(f".developer ({len(self.developer)} results)")available = ", ".join(parts) if parts else ".web, .news, or .images"raise AttributeError(f"SearchData has no '.data'. Results are grouped by source: {available}"