Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
search: serve developer category results inside web
Karan Lokchandani3c63d3ca7713
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.tsindex dbc267172..da202c891 100644--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts@@ -1124,9 +1124,18 @@ describe("V2 Types Validation", () => {expect(searchRequestSchema.parse({query: "test",- categories: ["docs", "github", "developer_index"],+ categories: ["docs", "developer_index"],}).categories,- ).toEqual([{ type: "developer" }, { type: "github" }]);+ ).toEqual([{ type: "developer" }]);++ // Developer (and its aliases) is exclusive: combining with any other+ // category is rejected at the schema.+ expect(() =>+ searchRequestSchema.parse({+ query: "test",+ categories: ["docs", "github", "developer_index"],+ }),+ ).toThrow(/cannot be combined/);});it("should reject developer alias params and unknown categories", () => {diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.tsindex 1037db4fc..6b2f93c4d 100644--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts@@ -72,25 +72,26 @@ const TEAM_ID = "11111111-1111-1111-1111-111111111111";const developerResults = [{+ id: "readme:example/repo",url: "https://github.com/example/repo",title: "example/repo",description: "a repo",- position: 1,- category: "developer",+ passages: [{ text: "a repo" }],},{+ id: "readme:example/other",url: "https://github.com/example/other",title: "example/other",description: "another repo",- position: 2,- category: "developer",+ passages: [{ text: "another repo" }],},];function executeResult(overrides: Record<string, any> = {}) {return {- response: { web: [], developer: developerResults },+ response: { web: developerResults },totalResultsCount: 2,+ developerResultsCount: 2,searchCredits: 2,scrapeCredits: 0,totalCredits: 2,@@ -269,7 +270,11 @@ describe("developer category code_searches ledger", () => {it("records zero results when the developer arm returns nothing", async () => {mockExecuteSearch.mockResolvedValue(- executeResult({ response: { web: [] }, totalResultsCount: 0 }),+ executeResult({+ response: { web: [] },+ totalResultsCount: 0,+ developerResultsCount: 0,+ }),);const req = makeReq({ query: "http client", categories: ["developer"] });const res = makeRes();@@ -295,12 +300,15 @@ describe("developer category code_searches ledger", () => {expect(res.status).toHaveBeenCalledWith(200);const body = res.json.mock.calls[0][0];expect(body.success).toBe(true);- expect(body.data.developer).toEqual(developerResults);+ expect(body.data.web).toEqual(developerResults);+ expect(body.data).not.toHaveProperty("developer");expect(body.creditsUsed).toBe(2);});it("keeps the developer category for a team with no flags", async () => {- mockExecuteSearch.mockResolvedValue(executeResult({ response: { web: [] } }));+ mockExecuteSearch.mockResolvedValue(+ executeResult({ response: { web: [] } }),+ );const req = makeReq({ query: "http client", categories: ["developer"] });req.acuc = { api_key_id: 7, flags: null }; // keyless-equivalent: no org flagsconst res = makeRes();diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex d16e43060..543566c7f 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -327,7 +327,7 @@ export async function searchController(via: "search_category",},response: null,- num_results: result.response.developer?.length ?? 0,+ num_results: result.developerResultsCount,time_taken: timeTakenInSeconds,// Ensure preview-mode searches don't get a non-zero credits_cost// in the research ledger when preview tokens are used.diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 2dd98a199..54233c799 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -2165,6 +2165,18 @@ export const searchRequestSchema = zx => !(x.includeDomains?.length && x.excludeDomains?.length),"includeDomains and excludeDomains cannot both be specified",)+ .refine(+ x => {+ const categories = x.categories ?? [];+ const hasDeveloper = categories.some(category =>+ typeof category === "string"+ ? category === "developer"+ : category.type === "developer",+ );+ return !hasDeveloper || categories.length === 1;+ },+ "the developer category cannot be combined with other categories",+ ).refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts).transform(x => {const country =diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.tsindex 130add82c..0d5dac8ee 100644--- a/apps/api/src/lib/entities.ts+++ b/apps/api/src/lib/entities.ts@@ -167,7 +167,6 @@ export interface SearchV2Response {web?: WebSearchResult[];images?: ImageSearchResult[];news?: NewsSearchResult[];- developer?: WebSearchResult[];}export interface ScrapeActionContent {diff --git a/apps/api/src/search/developer.test.ts b/apps/api/src/search/developer.test.tsindex 55e5a9869..b8902d0c9 100644--- a/apps/api/src/search/developer.test.ts+++ b/apps/api/src/search/developer.test.ts@@ -58,18 +58,21 @@ describe("searchDeveloperCategory", () => {expect(firstCall()[1].headers).toEqual({ "firecrawl-team-id": "t1" });});- it("maps upstream results into web result shape", async () => {+ it("maps results to the exact web schema and leaks nothing upstream", async () => {+ const license = { state: "licensed", spdx_id: "MIT" };fetchMock.mockResolvedValue(upstreamOk({results: [{- id: "abc",- type: "issue",+ id: "issue:a/b#1",url: "https://github.com/a/b/issues/1",title: "Retries drop the backoff",- passages: [{ text: "first passage" }],+ passages: [+ { text: "first passage", citation_url: "https://example.com" },+ ],+ license,},- { id: "def", type: "readme", url: "", passages: [] },+ { id: "readme:a/b", url: "", passages: [] },],}),);@@ -79,7 +82,9 @@ describe("searchDeveloperCategory", () => {logger,);- expect(results).toEqual([+ // Exact WebSearchResult shape: the upstream id/passages/license must not+ // appear on the wire; the first passage becomes the description.+ expect(results).toStrictEqual([{url: "https://github.com/a/b/issues/1",title: "Retries drop the backoff",diff --git a/apps/api/src/search/developer.ts b/apps/api/src/search/developer.tsindex fac7ceef7..78e5c6e51 100644--- a/apps/api/src/search/developer.ts+++ b/apps/api/src/search/developer.ts@@ -38,6 +38,9 @@ export async function searchDeveloperCategory(const body: any = await upstream.json();const results: any[] = Array.isArray(body?.results) ? body.results : [];+ // Exact WebSearchResult shape: developer results are indistinguishable+ // from ordinary web results on the wire. Passage text serves as the+ // description; nothing from the upstream payload leaks through.return results.slice(0, options.limit).map((result, index) => ({diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.tsnew file mode 100644index 000000000..92b97ca93--- /dev/null+++ b/apps/api/src/search/execute.test.ts@@ -0,0 +1,142 @@+const mocks = vi.hoisted(() => ({+ search: vi.fn(),+ searchDeveloperCategory: vi.fn(),+ checkUrlsAgainstThreatPolicy: vi.fn(),+}));++vi.mock("./v2", () => ({ search: mocks.search }));+vi.mock("./developer", () => ({+ wantsDeveloperCategory: (categories?: Array<string | { type: string }>) =>+ (categories ?? []).some(category =>+ typeof category === "string"+ ? category === "developer"+ : category.type === "developer",+ ),+ searchDeveloperCategory: mocks.searchDeveloperCategory,+}));+vi.mock("./scrape", () => ({+ getItemsToScrape: vi.fn(() => []),+ scrapeSearchResults: vi.fn(),+ mergeScrapedContent: vi.fn(),+ calculateScrapeCredits: vi.fn(() => 0),+}));+vi.mock("./highlights", () => ({+ highlightsEnvReady: () => false,+ runIndexedSearchHighlights: vi.fn(),+ searchHighlightsMode: vi.fn(),+}));+vi.mock("../lib/tracking", () => ({+ trackSearchRequest: vi.fn(async () => {}),+ trackSearchResults: vi.fn(async () => {}),+}));+vi.mock("../lib/threat-protection/request", () => ({+ checkUrlsAgainstThreatPolicy: mocks.checkUrlsAgainstThreatPolicy,+}));+vi.mock("../lib/scrape-billing", () => ({+ calculateThreatScanCredits: vi.fn(() => 0),+}));++import { executeSearch } from "./execute";+import { searchRequestSchema } from "../controllers/v2/types";++const logger = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+} as any;++const context = {+ teamId: "team-1",+ origin: "api",+ apiKeyId: 1,+ flags: {},+ requestId: "request-1",+ jobId: "job-1",+ apiVersion: "v2",+};++// searchDeveloperCategory returns the exact WebSearchResult shape.+const developerResult = {+ url: "https://github.com/firecrawl/firecrawl/issues/1",+ title: "Retry requests",+ description: "Use exponential backoff.",+ position: 1,+ category: "developer",+};++function options(categories: Array<{ type: string }>) {+ return {+ query: "retries",+ limit: 10,+ sources: [{ type: "web" }],+ categories,+ timeout: 1_000,+ } as any;+}++beforeEach(() => {+ vi.clearAllMocks();+ mocks.searchDeveloperCategory.mockResolvedValue([developerResult]);+});++describe("executeSearch developer category", () => {+ it("returns sole developer-category results in web without running SERP", async () => {+ const result = await executeSearch(+ options([{ type: "developer" }]),+ context,+ logger,+ );++ expect(mocks.search).not.toHaveBeenCalled();+ expect(result.response).toEqual({ web: [developerResult] });+ expect(result.response).not.toHaveProperty("developer");+ expect(result.developerResultsCount).toBe(1);+ });+++ it("filters blocked developer results via threat protection and renumbers", async () => {+ mocks.searchDeveloperCategory.mockResolvedValue([+ { url: "https://ok.example/a", title: "A", description: "", position: 1, category: "developer" },+ { url: "https://blocked.example/b", title: "B", description: "", position: 2, category: "developer" },+ { url: "https://ok.example/c", title: "C", description: "", position: 3, category: "developer" },+ ]);+ mocks.checkUrlsAgainstThreatPolicy.mockResolvedValue({+ decisionsByUrl: new Map([+ ["https://blocked.example/b", { allowed: false }],+ ]),+ });++ const result = await executeSearch(+ options([{ type: "developer" }]),+ { ...context, threatProtectionPolicy: { mode: "block" } } as any,+ logger,+ );++ expect(mocks.checkUrlsAgainstThreatPolicy).toHaveBeenCalledTimes(1);+ expect(mocks.search).not.toHaveBeenCalled();+ expect((result.response.web ?? []).map(x => [x.url, x.position])).toEqual([+ ["https://ok.example/a", 1],+ ["https://ok.example/c", 2],+ ]);+ });++ it("rejects developer combined with other categories at the schema", () => {+ const mixed = searchRequestSchema.safeParse({+ query: "retries",+ categories: ["developer", "github"],+ });+ expect(mixed.success).toBe(false);+ if (!mixed.success) {+ expect(JSON.stringify(mixed.error.issues)).toContain(+ "cannot be combined",+ );+ }++ const sole = searchRequestSchema.safeParse({+ query: "retries",+ categories: ["developer"],+ });+ expect(sole.success).toBe(true);+ });+});diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 8a74a59bf..5867d00d9 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -68,12 +68,22 @@ interface SearchContext {interface SearchExecuteResult {response: SearchV2Response;totalResultsCount: number;+ developerResultsCount: number;searchCredits: number;scrapeCredits: number;totalCredits: number;shouldScrape: boolean;}+function hasOnlyDeveloperCategory(categories?: CategoryOption[]): boolean {+ if (!categories?.length) return false;+ return categories.every(category =>+ typeof category === "string"+ ? category === "developer"+ : category.type === "developer",+ );+}+export async function executeSearch(options: SearchOptions,context: SearchContext,@@ -106,34 +116,33 @@ export async function executeSearch(},);- const developerResults = wantsDeveloperCategory(categories)+ const wantsDeveloper = wantsDeveloperCategory(categories);+ const developerResultsPromise = wantsDeveloper? searchDeveloperCategory({ query, limit, teamId, timeout: options.timeout },logger,): null;- const searchResponse = (await search({- query: searchQuery,- logger,- advanced: false,- num_results: num_results_buffer,- tbs: options.tbs,- filter: options.filter,- lang: options.lang,- country: options.country,- location: options.location,- safe: options.safe,- type: searchTypes,- enterprise: options.enterprise,- })) as SearchV2Response;-- if (developerResults) {- const developer = await developerResults;- if (developer.length > 0) {- searchResponse.developer = developer;- }- }+ const searchResponse = hasOnlyDeveloperCategory(categories)+ ? ({} as SearchV2Response)+ : ((await search({+ query: searchQuery,+ logger,+ advanced: false,+ num_results: num_results_buffer,+ tbs: options.tbs,+ filter: options.filter,+ lang: options.lang,+ country: options.country,+ location: options.location,+ safe: options.safe,+ type: searchTypes,+ enterprise: options.enterprise,+ })) as SearchV2Response);+ let developerResults = developerResultsPromise+ ? await developerResultsPromise+ : [];// Threat protection: remove blocked results entirely β before// slicing/counting, before scraping, and before returning. Checks are@@ -147,7 +156,7 @@ export async function executeSearch(...(searchResponse.web ?? []).map(x => x.url),...(searchResponse.news ?? []).map(x => x.url),...(searchResponse.images ?? []).map(x => x.url),- ...(searchResponse.developer ?? []).map(x => x.url),+ ...developerResults.map(x => x.url),].filter((x): x is string => !!x);if (urlsToCheck.length > 0) {@@ -173,11 +182,7 @@ export async function executeSearch(isAllowed(x.url),);}- if (searchResponse.developer) {- searchResponse.developer = searchResponse.developer.filter(x =>- isAllowed(x.url),- );- }+ developerResults = developerResults.filter(x => isAllowed(x.url));}}@@ -220,9 +225,8 @@ export async function executeSearch(totalResultsCount += searchResponse.news.length;}- if (searchResponse.developer && searchResponse.developer.length > 0) {- totalResultsCount += searchResponse.developer.length;- }+ const developerResultsCount = developerResults.length;+ totalResultsCount += developerResultsCount;const isZDR = options.enterprise?.includes("zdr");const creditsPerTenResults = isZDR ? 10 : 2;@@ -321,6 +325,16 @@ export async function executeSearch(}}+ if (wantsDeveloper) {+ // The developer category is exclusive (schema-enforced), so these are+ // the only results: they ARE the web group. Threat filtering above may+ // have removed entries, so renumber the survivors.+ searchResponse.web = developerResults.map((result, index) => ({+ ...result,+ position: index + 1,+ }));+ }+const scrapeFormats = scrapeOptions?.formats? scrapeOptions.formats.map((f: any) =>typeof f === "string" ? f : f.type,@@ -362,6 +376,7 @@ export async function executeSearch(return {response: searchResponse,totalResultsCount,+ developerResultsCount,searchCredits,scrapeCredits,totalCredits: searchCredits + scrapeCredits,diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex ae8d88c7a..a0b034a04 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -183,7 +183,8 @@ const scraped = await app.search('firecrawl changelog', {});```-Results are grouped by source: `.web`, `.news`, `.images` and `.developer`.+Results are grouped by source: `.web`, `.news` and `.images`. Developer+category results are served inside `.web`.Use `categories` to narrow web search to a kind of site:@@ -202,8 +203,8 @@ const results = await app.search('nanopore basecalling accuracy', {Use `developerSearch` for the dedicated developer index and its complete filterand response contract. Generic `search(..., { categories: ['developer'] })`-returns only the first passage as a description and does not accept these-filters.+returns developer results inside `.web` in the ordinary web-result shape+(first passage as the description) and does not accept these filters.```jsconst evidence = await app.developerSearch('configure retry backoff', {diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.tsindex 43c3977bd..b3a1d9ac0 100644--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts@@ -1,8 +1,10 @@import { describe, expect, jest, test } from "@jest/globals";import { search } from "../../../v2/methods/search";+import type { SearchResultWeb } from "../../../v2/types";-const developerResult = {- title: "https://github.com/firecrawl/firecrawl/issues/1",+// Developer results arrive in the exact web schema.+const developerResult: SearchResultWeb = {+ title: "Retry requests",url: "https://github.com/firecrawl/firecrawl/issues/1",description: "passage",position: 1,@@ -28,38 +30,26 @@ describe("v2 search developer category", () => {);});- test("returns developer results", async () => {- const http = httpWith({- web: [{ title: "W", url: "http://a.com" }],- developer: [developerResult],- });+ test("returns developer results inside web", async () => {+ const http = httpWith({ web: [developerResult] });const result = await search(http, {query: "firecrawl",categories: [{ type: "developer" }],});- expect(result.web).toHaveLength(1);- expect(result.developer).toHaveLength(1);- expect(result.developer?.[0]).toEqual(developerResult);- });-- test("omits developer when the response has none", async () => {- const http = httpWith({ web: [] });-- const result = await search(http, { query: "firecrawl" });-- expect(result.developer).toBeUndefined();+ expect(result.web).toEqual([developerResult]);+ expect(result).not.toHaveProperty("developer");});- test("lists developer in the .data error", async () => {- const http = httpWith({ developer: [developerResult] });+ test("lists developer results as web results in the .data error", async () => {+ const http = httpWith({ web: [developerResult] });const result = await search(http, {query: "firecrawl",categories: ["developer"],});- expect(() => (result as any).data).toThrow(/\.developer \(1 results\)/);+ expect(() => (result as any).data).toThrow(/\.web \(1 results\)/);});});diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.tsindex 89b6506b9..9dc82c4c4 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts@@ -102,17 +102,16 @@ export async function search(if (data.news) out.news = transformArray<SearchResultNews>(data.news);if (data.images)out.images = transformArray<SearchResultImages>(data.images);- if (data.developer)- out.developer = transformArray<SearchResultWeb>(data.developer);Object.defineProperty(out, "data", {get() {const parts: string[] = [];if (out.web?.length) parts.push(`.web (${out.web.length} results)`);if (out.news?.length) parts.push(`.news (${out.news.length} results)`);- if (out.images?.length) parts.push(`.images (${out.images.length} results)`);- if (out.developer?.length)- parts.push(`.developer (${out.developer.length} results)`);- const available = parts.length ? parts.join(", ") : ".web, .news, or .images";+ if (out.images?.length)+ parts.push(`.images (${out.images.length} results)`);+ const available = parts.length+ ? parts.join(", ")+ : ".web, .news, or .images";throw new Error(`SearchData has no '.data'. Results are grouped by source: ${available}`,);diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex 6241f6676..1c6b76e2f 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -808,7 +808,6 @@ export interface SearchData {web?: Array<SearchResultWeb | Document>;news?: Array<SearchResultNews | Document>;images?: Array<SearchResultImages | Document>;- developer?: Array<SearchResultWeb | Document>;}/**@@ -821,8 +820,8 @@ export interface SearchData {* ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns* ordinary web page results from those domains, **not** paper records.* - `pdf` β restrict results to PDFs (adds `filetype:pdf`).- * - `developer` β add developer results (issues, pull requests, READMEs and- * documentation) under `.developer`.+ * - `developer` β developer-index results (issues, pull requests, READMEs and+ * documentation) served in `web`; cannot be combined with other categories.** β οΈ `categories: ["research"]` is **not** Firecrawl's research paper index.* To search papers themselves β ~43M abstracts, roughly 90% biomedicaldiff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.pyindex 01eb79c76..6ca2858ca 100644--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py@@ -13,61 +13,51 @@ def _response(data):return response-DEVELOPER_PAYLOAD = {- "web": [{"title": "W", "url": "http://a.com", "description": "d"}],- "developer": [- {- "title": "https://github.com/firecrawl/firecrawl/issues/1",- "url": "https://github.com/firecrawl/firecrawl/issues/1",- "description": "passage",- "position": 1,- "category": "developer",- }- ],+# Developer results arrive in the exact web schema; nothing extra on the wire.+DEVELOPER_RESULT = {+ "title": "Retry requests",+ "url": "https://github.com/firecrawl/firecrawl/issues/1",+ "description": "passage",+ "position": 1,+ "category": "developer",}class TestSearchDeveloperResults:- """Unit tests for the developer category results."""+ """Unit tests for developer category results in the web group."""- def test_developer_results_are_parsed(self):+ def test_developer_results_are_parsed_inside_web(self):client = Mock()- client.post.return_value = _response(DEVELOPER_PAYLOAD)+ client.post.return_value = _response({"web": [DEVELOPER_RESULT]})result = search(client, SearchRequest(query="test", categories=["developer"]))assert len(result.web) == 1- assert len(result.developer) == 1- assert isinstance(result.developer[0], SearchResultWeb)- assert result.developer[0].url == "https://github.com/firecrawl/firecrawl/issues/1"- assert result.developer[0].position == 1- assert result.developer[0].category == "developer"+ assert isinstance(result.web[0], SearchResultWeb)+ assert result.web[0].url.endswith("/issues/1")+ assert result.web[0].category == "developer"+ assert not hasattr(result.web[0], "passages")+ assert not hasattr(result, "developer")- def test_developer_absent_when_not_returned(self):+ def test_data_property_lists_developer_results_as_web(self):client = Mock()- client.post.return_value = _response({"web": []})-- result = search(client, SearchRequest(query="test"))-- assert result.developer is None-- def test_data_property_lists_developer(self):- client = Mock()- client.post.return_value = _response({"developer": DEVELOPER_PAYLOAD["developer"]})+ client.post.return_value = _response({"web": [DEVELOPER_RESULT]})result = search(client, SearchRequest(query="test", categories=["developer"]))- with pytest.raises(AttributeError, match=r"\.developer \(1 results\)"):+ with pytest.raises(AttributeError, match=r"\.web \(1 results\)"):result.data@pytest.mark.asyncio- async def test_developer_results_are_parsed_async(self):+ async def test_developer_results_are_parsed_inside_web_async(self):client = Mock()- client.post = AsyncMock(return_value=_response(DEVELOPER_PAYLOAD))+ client.post = AsyncMock(return_value=_response({"web": [DEVELOPER_RESULT]}))result = await search_async(client, SearchRequest(query="test", categories=["developer"]))- assert len(result.developer) == 1- assert result.developer[0].url == "https://github.com/firecrawl/firecrawl/issues/1"+ assert len(result.web) == 1+ assert isinstance(result.web[0], SearchResultWeb)+ assert result.web[0].category == "developer"+ assert not hasattr(result, "developer")diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/search.py b/apps/python-sdk/firecrawl/v2/methods/aio/search.pyindex 6dbe2fd9b..02f5e0bb0 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/search.py@@ -47,8 +47,6 @@ async def search(out.news = _transform_array(data["news"], SearchResultNews)if "images" in data:out.images = _transform_array(data["images"], SearchResultImages)- if "developer" in data:- out.developer = _transform_array(data["developer"], SearchResultWeb)return outexcept Exception as err:if hasattr(err, "response"):diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.pyindex 0f23ff00c..3b771bce1 100644--- a/apps/python-sdk/firecrawl/v2/methods/search.py+++ b/apps/python-sdk/firecrawl/v2/methods/search.py@@ -42,8 +42,6 @@ def search(out.news = _transform_array(data["news"], SearchResultNews)if "images" in data:out.images = _transform_array(data["images"], SearchResultImages)- if "developer" in data:- out.developer = _transform_array(data["developer"], SearchResultWeb)return outexcept Exception as err:# If the error is an HTTP error from requests, handle itdiff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 1fe6bf4bc..4eeab4e9f 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -619,8 +619,8 @@ class Category(BaseModel):website/domain filter and it returns ordinary web page results for thosedomains β **not** paper records.- "pdf": Filter results to PDF files (adds filetype:pdf to search)- - "developer": Add developer results (issues, pull requests, READMEs and- documentation) under `.developer`+ - "developer": Developer-index results (issues, pull requests, READMEs and+ documentation) served in `web`; cannot be combined with other categories.. warning::``categories=["research"]`` is **not** Firecrawl's research paper index.@@ -1855,7 +1855,6 @@ class SearchData(BaseModel):web: Optional[List[Union[SearchResultWeb, Document]]] = Nonenews: Optional[List[Union[SearchResultNews, Document]]] = Noneimages: Optional[List[Union[SearchResultImages, Document]]] = None- developer: Optional[List[Union[SearchResultWeb, Document]]] = None@propertydef data(self):@@ -1866,8 +1865,6 @@ class SearchData(BaseModel):parts.append(f".news ({len(self.news)} results)")if self.images:parts.append(f".images ({len(self.images)} results)")- if self.developer:- parts.append(f".developer ({len(self.developer)} results)")available = ", ".join(parts) if parts else ".web, .news, or .images"raise AttributeError(f"SearchData has no '.data'. Results are grouped by source: {available}"