Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

search: serve developer category results inside web

Karan Lokchandani3c63d3ca7713
diff --git a/apps/api/src/__tests__/snips/v2/types-validation.test.ts b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
index dbc267172..da202c891 100644
--- a/apps/api/src/__tests__/snips/v2/types-validation.test.ts
+++ b/apps/api/src/__tests__/snips/v2/types-validation.test.ts
@@ -1124,9 +1124,18 @@ describe("V2 Types Validation", () => {
expect(
searchRequestSchema.parse({
query: "test",
- categories: ["docs", "github", "developer_index"],
+ categories: ["docs", "developer_index"],
}).categories,
- ).toEqual([{ type: "developer" }, { type: "github" }]);
+ ).toEqual([{ type: "developer" }]);
+
+ // Developer (and its aliases) is exclusive: combining with any other
+ // category is rejected at the schema.
+ expect(() =>
+ searchRequestSchema.parse({
+ query: "test",
+ categories: ["docs", "github", "developer_index"],
+ }),
+ ).toThrow(/cannot be combined/);
});
it("should reject developer alias params and unknown categories", () => {
diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
index 1037db4fc..6b2f93c4d 100644
--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts
@@ -72,25 +72,26 @@ const TEAM_ID = "11111111-1111-1111-1111-111111111111";
const developerResults = [
{
+ id: "readme:example/repo",
url: "https://github.com/example/repo",
title: "example/repo",
description: "a repo",
- position: 1,
- category: "developer",
+ passages: [{ text: "a repo" }],
},
{
+ id: "readme:example/other",
url: "https://github.com/example/other",
title: "example/other",
description: "another repo",
- position: 2,
- category: "developer",
+ passages: [{ text: "another repo" }],
},
];
function executeResult(overrides: Record<string, any> = {}) {
return {
- response: { web: [], developer: developerResults },
+ response: { web: developerResults },
totalResultsCount: 2,
+ developerResultsCount: 2,
searchCredits: 2,
scrapeCredits: 0,
totalCredits: 2,
@@ -269,7 +270,11 @@ describe("developer category code_searches ledger", () => {
it("records zero results when the developer arm returns nothing", async () => {
mockExecuteSearch.mockResolvedValue(
- executeResult({ response: { web: [] }, totalResultsCount: 0 }),
+ executeResult({
+ response: { web: [] },
+ totalResultsCount: 0,
+ developerResultsCount: 0,
+ }),
);
const req = makeReq({ query: "http client", categories: ["developer"] });
const res = makeRes();
@@ -295,12 +300,15 @@ describe("developer category code_searches ledger", () => {
expect(res.status).toHaveBeenCalledWith(200);
const body = res.json.mock.calls[0][0];
expect(body.success).toBe(true);
- expect(body.data.developer).toEqual(developerResults);
+ expect(body.data.web).toEqual(developerResults);
+ expect(body.data).not.toHaveProperty("developer");
expect(body.creditsUsed).toBe(2);
});
it("keeps the developer category for a team with no flags", async () => {
- mockExecuteSearch.mockResolvedValue(executeResult({ response: { web: [] } }));
+ mockExecuteSearch.mockResolvedValue(
+ executeResult({ response: { web: [] } }),
+ );
const req = makeReq({ query: "http client", categories: ["developer"] });
req.acuc = { api_key_id: 7, flags: null }; // keyless-equivalent: no org flags
const res = makeRes();
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index d16e43060..543566c7f 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -327,7 +327,7 @@ export async function searchController(
via: "search_category",
},
response: null,
- num_results: result.response.developer?.length ?? 0,
+ num_results: result.developerResultsCount,
time_taken: timeTakenInSeconds,
// Ensure preview-mode searches don't get a non-zero credits_cost
// in the research ledger when preview tokens are used.
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 2dd98a199..54233c799 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -2165,6 +2165,18 @@ export const searchRequestSchema = z
x => !(x.includeDomains?.length && x.excludeDomains?.length),
"includeDomains and excludeDomains cannot both be specified",
)
+ .refine(
+ x => {
+ const categories = x.categories ?? [];
+ const hasDeveloper = categories.some(category =>
+ typeof category === "string"
+ ? category === "developer"
+ : category.type === "developer",
+ );
+ return !hasDeveloper || categories.length === 1;
+ },
+ "the developer category cannot be combined with other categories",
+ )
.refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts)
.transform(x => {
const country =
diff --git a/apps/api/src/lib/entities.ts b/apps/api/src/lib/entities.ts
index 130add82c..0d5dac8ee 100644
--- a/apps/api/src/lib/entities.ts
+++ b/apps/api/src/lib/entities.ts
@@ -167,7 +167,6 @@ export interface SearchV2Response {
web?: WebSearchResult[];
images?: ImageSearchResult[];
news?: NewsSearchResult[];
- developer?: WebSearchResult[];
}
export interface ScrapeActionContent {
diff --git a/apps/api/src/search/developer.test.ts b/apps/api/src/search/developer.test.ts
index 55e5a9869..b8902d0c9 100644
--- a/apps/api/src/search/developer.test.ts
+++ b/apps/api/src/search/developer.test.ts
@@ -58,18 +58,21 @@ describe("searchDeveloperCategory", () => {
expect(firstCall()[1].headers).toEqual({ "firecrawl-team-id": "t1" });
});
- it("maps upstream results into web result shape", async () => {
+ it("maps results to the exact web schema and leaks nothing upstream", async () => {
+ const license = { state: "licensed", spdx_id: "MIT" };
fetchMock.mockResolvedValue(
upstreamOk({
results: [
{
- id: "abc",
- type: "issue",
+ id: "issue:a/b#1",
url: "https://github.com/a/b/issues/1",
title: "Retries drop the backoff",
- passages: [{ text: "first passage" }],
+ passages: [
+ { text: "first passage", citation_url: "https://example.com" },
+ ],
+ license,
},
- { id: "def", type: "readme", url: "", passages: [] },
+ { id: "readme:a/b", url: "", passages: [] },
],
}),
);
@@ -79,7 +82,9 @@ describe("searchDeveloperCategory", () => {
logger,
);
- expect(results).toEqual([
+ // Exact WebSearchResult shape: the upstream id/passages/license must not
+ // appear on the wire; the first passage becomes the description.
+ expect(results).toStrictEqual([
{
url: "https://github.com/a/b/issues/1",
title: "Retries drop the backoff",
diff --git a/apps/api/src/search/developer.ts b/apps/api/src/search/developer.ts
index fac7ceef7..78e5c6e51 100644
--- a/apps/api/src/search/developer.ts
+++ b/apps/api/src/search/developer.ts
@@ -38,6 +38,9 @@ export async function searchDeveloperCategory(
const body: any = await upstream.json();
const results: any[] = Array.isArray(body?.results) ? body.results : [];
+ // Exact WebSearchResult shape: developer results are indistinguishable
+ // from ordinary web results on the wire. Passage text serves as the
+ // description; nothing from the upstream payload leaks through.
return results
.slice(0, options.limit)
.map((result, index) => ({
diff --git a/apps/api/src/search/execute.test.ts b/apps/api/src/search/execute.test.ts
new file mode 100644
index 000000000..92b97ca93
--- /dev/null
+++ b/apps/api/src/search/execute.test.ts
@@ -0,0 +1,142 @@
+const mocks = vi.hoisted(() => ({
+ search: vi.fn(),
+ searchDeveloperCategory: vi.fn(),
+ checkUrlsAgainstThreatPolicy: vi.fn(),
+}));
+
+vi.mock("./v2", () => ({ search: mocks.search }));
+vi.mock("./developer", () => ({
+ wantsDeveloperCategory: (categories?: Array<string | { type: string }>) =>
+ (categories ?? []).some(category =>
+ typeof category === "string"
+ ? category === "developer"
+ : category.type === "developer",
+ ),
+ searchDeveloperCategory: mocks.searchDeveloperCategory,
+}));
+vi.mock("./scrape", () => ({
+ getItemsToScrape: vi.fn(() => []),
+ scrapeSearchResults: vi.fn(),
+ mergeScrapedContent: vi.fn(),
+ calculateScrapeCredits: vi.fn(() => 0),
+}));
+vi.mock("./highlights", () => ({
+ highlightsEnvReady: () => false,
+ runIndexedSearchHighlights: vi.fn(),
+ searchHighlightsMode: vi.fn(),
+}));
+vi.mock("../lib/tracking", () => ({
+ trackSearchRequest: vi.fn(async () => {}),
+ trackSearchResults: vi.fn(async () => {}),
+}));
+vi.mock("../lib/threat-protection/request", () => ({
+ checkUrlsAgainstThreatPolicy: mocks.checkUrlsAgainstThreatPolicy,
+}));
+vi.mock("../lib/scrape-billing", () => ({
+ calculateThreatScanCredits: vi.fn(() => 0),
+}));
+
+import { executeSearch } from "./execute";
+import { searchRequestSchema } from "../controllers/v2/types";
+
+const logger = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+} as any;
+
+const context = {
+ teamId: "team-1",
+ origin: "api",
+ apiKeyId: 1,
+ flags: {},
+ requestId: "request-1",
+ jobId: "job-1",
+ apiVersion: "v2",
+};
+
+// searchDeveloperCategory returns the exact WebSearchResult shape.
+const developerResult = {
+ url: "https://github.com/firecrawl/firecrawl/issues/1",
+ title: "Retry requests",
+ description: "Use exponential backoff.",
+ position: 1,
+ category: "developer",
+};
+
+function options(categories: Array<{ type: string }>) {
+ return {
+ query: "retries",
+ limit: 10,
+ sources: [{ type: "web" }],
+ categories,
+ timeout: 1_000,
+ } as any;
+}
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.searchDeveloperCategory.mockResolvedValue([developerResult]);
+});
+
+describe("executeSearch developer category", () => {
+ it("returns sole developer-category results in web without running SERP", async () => {
+ const result = await executeSearch(
+ options([{ type: "developer" }]),
+ context,
+ logger,
+ );
+
+ expect(mocks.search).not.toHaveBeenCalled();
+ expect(result.response).toEqual({ web: [developerResult] });
+ expect(result.response).not.toHaveProperty("developer");
+ expect(result.developerResultsCount).toBe(1);
+ });
+
+
+ it("filters blocked developer results via threat protection and renumbers", async () => {
+ mocks.searchDeveloperCategory.mockResolvedValue([
+ { url: "https://ok.example/a", title: "A", description: "", position: 1, category: "developer" },
+ { url: "https://blocked.example/b", title: "B", description: "", position: 2, category: "developer" },
+ { url: "https://ok.example/c", title: "C", description: "", position: 3, category: "developer" },
+ ]);
+ mocks.checkUrlsAgainstThreatPolicy.mockResolvedValue({
+ decisionsByUrl: new Map([
+ ["https://blocked.example/b", { allowed: false }],
+ ]),
+ });
+
+ const result = await executeSearch(
+ options([{ type: "developer" }]),
+ { ...context, threatProtectionPolicy: { mode: "block" } } as any,
+ logger,
+ );
+
+ expect(mocks.checkUrlsAgainstThreatPolicy).toHaveBeenCalledTimes(1);
+ expect(mocks.search).not.toHaveBeenCalled();
+ expect((result.response.web ?? []).map(x => [x.url, x.position])).toEqual([
+ ["https://ok.example/a", 1],
+ ["https://ok.example/c", 2],
+ ]);
+ });
+
+ it("rejects developer combined with other categories at the schema", () => {
+ const mixed = searchRequestSchema.safeParse({
+ query: "retries",
+ categories: ["developer", "github"],
+ });
+ expect(mixed.success).toBe(false);
+ if (!mixed.success) {
+ expect(JSON.stringify(mixed.error.issues)).toContain(
+ "cannot be combined",
+ );
+ }
+
+ const sole = searchRequestSchema.safeParse({
+ query: "retries",
+ categories: ["developer"],
+ });
+ expect(sole.success).toBe(true);
+ });
+});
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 8a74a59bf..5867d00d9 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -68,12 +68,22 @@ interface SearchContext {
interface SearchExecuteResult {
response: SearchV2Response;
totalResultsCount: number;
+ developerResultsCount: number;
searchCredits: number;
scrapeCredits: number;
totalCredits: number;
shouldScrape: boolean;
}
+function hasOnlyDeveloperCategory(categories?: CategoryOption[]): boolean {
+ if (!categories?.length) return false;
+ return categories.every(category =>
+ typeof category === "string"
+ ? category === "developer"
+ : category.type === "developer",
+ );
+}
+
export async function executeSearch(
options: SearchOptions,
context: SearchContext,
@@ -106,34 +116,33 @@ export async function executeSearch(
},
);
- const developerResults = wantsDeveloperCategory(categories)
+ const wantsDeveloper = wantsDeveloperCategory(categories);
+ const developerResultsPromise = wantsDeveloper
? searchDeveloperCategory(
{ query, limit, teamId, timeout: options.timeout },
logger,
)
: null;
- const searchResponse = (await search({
- query: searchQuery,
- logger,
- advanced: false,
- num_results: num_results_buffer,
- tbs: options.tbs,
- filter: options.filter,
- lang: options.lang,
- country: options.country,
- location: options.location,
- safe: options.safe,
- type: searchTypes,
- enterprise: options.enterprise,
- })) as SearchV2Response;
-
- if (developerResults) {
- const developer = await developerResults;
- if (developer.length > 0) {
- searchResponse.developer = developer;
- }
- }
+ const searchResponse = hasOnlyDeveloperCategory(categories)
+ ? ({} as SearchV2Response)
+ : ((await search({
+ query: searchQuery,
+ logger,
+ advanced: false,
+ num_results: num_results_buffer,
+ tbs: options.tbs,
+ filter: options.filter,
+ lang: options.lang,
+ country: options.country,
+ location: options.location,
+ safe: options.safe,
+ type: searchTypes,
+ enterprise: options.enterprise,
+ })) as SearchV2Response);
+ let developerResults = developerResultsPromise
+ ? await developerResultsPromise
+ : [];
// Threat protection: remove blocked results entirely β€” before
// slicing/counting, before scraping, and before returning. Checks are
@@ -147,7 +156,7 @@ export async function executeSearch(
...(searchResponse.web ?? []).map(x => x.url),
...(searchResponse.news ?? []).map(x => x.url),
...(searchResponse.images ?? []).map(x => x.url),
- ...(searchResponse.developer ?? []).map(x => x.url),
+ ...developerResults.map(x => x.url),
].filter((x): x is string => !!x);
if (urlsToCheck.length > 0) {
@@ -173,11 +182,7 @@ export async function executeSearch(
isAllowed(x.url),
);
}
- if (searchResponse.developer) {
- searchResponse.developer = searchResponse.developer.filter(x =>
- isAllowed(x.url),
- );
- }
+ developerResults = developerResults.filter(x => isAllowed(x.url));
}
}
@@ -220,9 +225,8 @@ export async function executeSearch(
totalResultsCount += searchResponse.news.length;
}
- if (searchResponse.developer && searchResponse.developer.length > 0) {
- totalResultsCount += searchResponse.developer.length;
- }
+ const developerResultsCount = developerResults.length;
+ totalResultsCount += developerResultsCount;
const isZDR = options.enterprise?.includes("zdr");
const creditsPerTenResults = isZDR ? 10 : 2;
@@ -321,6 +325,16 @@ export async function executeSearch(
}
}
+ if (wantsDeveloper) {
+ // The developer category is exclusive (schema-enforced), so these are
+ // the only results: they ARE the web group. Threat filtering above may
+ // have removed entries, so renumber the survivors.
+ searchResponse.web = developerResults.map((result, index) => ({
+ ...result,
+ position: index + 1,
+ }));
+ }
+
const scrapeFormats = scrapeOptions?.formats
? scrapeOptions.formats.map((f: any) =>
typeof f === "string" ? f : f.type,
@@ -362,6 +376,7 @@ export async function executeSearch(
return {
response: searchResponse,
totalResultsCount,
+ developerResultsCount,
searchCredits,
scrapeCredits,
totalCredits: searchCredits + scrapeCredits,
diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.md
index ae8d88c7a..a0b034a04 100644
--- a/apps/js-sdk/firecrawl/README.md
+++ b/apps/js-sdk/firecrawl/README.md
@@ -183,7 +183,8 @@ const scraped = await app.search('firecrawl changelog', {
});
```
-Results are grouped by source: `.web`, `.news`, `.images` and `.developer`.
+Results are grouped by source: `.web`, `.news` and `.images`. Developer
+category results are served inside `.web`.
Use `categories` to narrow web search to a kind of site:
@@ -202,8 +203,8 @@ const results = await app.search('nanopore basecalling accuracy', {
Use `developerSearch` for the dedicated developer index and its complete filter
and response contract. Generic `search(..., { categories: ['developer'] })`
-returns only the first passage as a description and does not accept these
-filters.
+returns developer results inside `.web` in the ordinary web-result shape
+(first passage as the description) and does not accept these filters.
```js
const evidence = await app.developerSearch('configure retry backoff', {
diff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts
index 43c3977bd..b3a1d9ac0 100644
--- a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts
+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/search-developer.unit.test.ts
@@ -1,8 +1,10 @@
import { describe, expect, jest, test } from "@jest/globals";
import { search } from "../../../v2/methods/search";
+import type { SearchResultWeb } from "../../../v2/types";
-const developerResult = {
- title: "https://github.com/firecrawl/firecrawl/issues/1",
+// Developer results arrive in the exact web schema.
+const developerResult: SearchResultWeb = {
+ title: "Retry requests",
url: "https://github.com/firecrawl/firecrawl/issues/1",
description: "passage",
position: 1,
@@ -28,38 +30,26 @@ describe("v2 search developer category", () => {
);
});
- test("returns developer results", async () => {
- const http = httpWith({
- web: [{ title: "W", url: "http://a.com" }],
- developer: [developerResult],
- });
+ test("returns developer results inside web", async () => {
+ const http = httpWith({ web: [developerResult] });
const result = await search(http, {
query: "firecrawl",
categories: [{ type: "developer" }],
});
- expect(result.web).toHaveLength(1);
- expect(result.developer).toHaveLength(1);
- expect(result.developer?.[0]).toEqual(developerResult);
- });
-
- test("omits developer when the response has none", async () => {
- const http = httpWith({ web: [] });
-
- const result = await search(http, { query: "firecrawl" });
-
- expect(result.developer).toBeUndefined();
+ expect(result.web).toEqual([developerResult]);
+ expect(result).not.toHaveProperty("developer");
});
- test("lists developer in the .data error", async () => {
- const http = httpWith({ developer: [developerResult] });
+ test("lists developer results as web results in the .data error", async () => {
+ const http = httpWith({ web: [developerResult] });
const result = await search(http, {
query: "firecrawl",
categories: ["developer"],
});
- expect(() => (result as any).data).toThrow(/\.developer \(1 results\)/);
+ expect(() => (result as any).data).toThrow(/\.web \(1 results\)/);
});
});
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/search.ts b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
index 89b6506b9..9dc82c4c4 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/search.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/search.ts
@@ -102,17 +102,16 @@ export async function search(
if (data.news) out.news = transformArray<SearchResultNews>(data.news);
if (data.images)
out.images = transformArray<SearchResultImages>(data.images);
- if (data.developer)
- out.developer = transformArray<SearchResultWeb>(data.developer);
Object.defineProperty(out, "data", {
get() {
const parts: string[] = [];
if (out.web?.length) parts.push(`.web (${out.web.length} results)`);
if (out.news?.length) parts.push(`.news (${out.news.length} results)`);
- if (out.images?.length) parts.push(`.images (${out.images.length} results)`);
- if (out.developer?.length)
- parts.push(`.developer (${out.developer.length} results)`);
- const available = parts.length ? parts.join(", ") : ".web, .news, or .images";
+ if (out.images?.length)
+ parts.push(`.images (${out.images.length} results)`);
+ const available = parts.length
+ ? parts.join(", ")
+ : ".web, .news, or .images";
throw new Error(
`SearchData has no '.data'. Results are grouped by source: ${available}`,
);
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index 6241f6676..1c6b76e2f 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -808,7 +808,6 @@ export interface SearchData {
web?: Array<SearchResultWeb | Document>;
news?: Array<SearchResultNews | Document>;
images?: Array<SearchResultImages | Document>;
- developer?: Array<SearchResultWeb | Document>;
}
/**
@@ -821,8 +820,8 @@ export interface SearchData {
* ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
* ordinary web page results from those domains, **not** paper records.
* - `pdf` β€” restrict results to PDFs (adds `filetype:pdf`).
- * - `developer` β€” add developer results (issues, pull requests, READMEs and
- * documentation) under `.developer`.
+ * - `developer` β€” developer-index results (issues, pull requests, READMEs and
+ * documentation) served in `web`; cannot be combined with other categories.
*
* ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
* To search papers themselves β€” ~43M abstracts, roughly 90% biomedical
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py
index 01eb79c76..6ca2858ca 100644
--- a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_search_response.py
@@ -13,61 +13,51 @@ def _response(data):
return response
-DEVELOPER_PAYLOAD = {
- "web": [{"title": "W", "url": "http://a.com", "description": "d"}],
- "developer": [
- {
- "title": "https://github.com/firecrawl/firecrawl/issues/1",
- "url": "https://github.com/firecrawl/firecrawl/issues/1",
- "description": "passage",
- "position": 1,
- "category": "developer",
- }
- ],
+# Developer results arrive in the exact web schema; nothing extra on the wire.
+DEVELOPER_RESULT = {
+ "title": "Retry requests",
+ "url": "https://github.com/firecrawl/firecrawl/issues/1",
+ "description": "passage",
+ "position": 1,
+ "category": "developer",
}
class TestSearchDeveloperResults:
- """Unit tests for the developer category results."""
+ """Unit tests for developer category results in the web group."""
- def test_developer_results_are_parsed(self):
+ def test_developer_results_are_parsed_inside_web(self):
client = Mock()
- client.post.return_value = _response(DEVELOPER_PAYLOAD)
+ client.post.return_value = _response({"web": [DEVELOPER_RESULT]})
result = search(client, SearchRequest(query="test", categories=["developer"]))
assert len(result.web) == 1
- assert len(result.developer) == 1
- assert isinstance(result.developer[0], SearchResultWeb)
- assert result.developer[0].url == "https://github.com/firecrawl/firecrawl/issues/1"
- assert result.developer[0].position == 1
- assert result.developer[0].category == "developer"
+ assert isinstance(result.web[0], SearchResultWeb)
+ assert result.web[0].url.endswith("/issues/1")
+ assert result.web[0].category == "developer"
+ assert not hasattr(result.web[0], "passages")
+ assert not hasattr(result, "developer")
- def test_developer_absent_when_not_returned(self):
+ def test_data_property_lists_developer_results_as_web(self):
client = Mock()
- client.post.return_value = _response({"web": []})
-
- result = search(client, SearchRequest(query="test"))
-
- assert result.developer is None
-
- def test_data_property_lists_developer(self):
- client = Mock()
- client.post.return_value = _response({"developer": DEVELOPER_PAYLOAD["developer"]})
+ client.post.return_value = _response({"web": [DEVELOPER_RESULT]})
result = search(client, SearchRequest(query="test", categories=["developer"]))
- with pytest.raises(AttributeError, match=r"\.developer \(1 results\)"):
+ with pytest.raises(AttributeError, match=r"\.web \(1 results\)"):
result.data
@pytest.mark.asyncio
- async def test_developer_results_are_parsed_async(self):
+ async def test_developer_results_are_parsed_inside_web_async(self):
client = Mock()
- client.post = AsyncMock(return_value=_response(DEVELOPER_PAYLOAD))
+ client.post = AsyncMock(return_value=_response({"web": [DEVELOPER_RESULT]}))
result = await search_async(
client, SearchRequest(query="test", categories=["developer"])
)
- assert len(result.developer) == 1
- assert result.developer[0].url == "https://github.com/firecrawl/firecrawl/issues/1"
+ assert len(result.web) == 1
+ assert isinstance(result.web[0], SearchResultWeb)
+ assert result.web[0].category == "developer"
+ assert not hasattr(result, "developer")
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/search.py b/apps/python-sdk/firecrawl/v2/methods/aio/search.py
index 6dbe2fd9b..02f5e0bb0 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/search.py
@@ -47,8 +47,6 @@ async def search(
out.news = _transform_array(data["news"], SearchResultNews)
if "images" in data:
out.images = _transform_array(data["images"], SearchResultImages)
- if "developer" in data:
- out.developer = _transform_array(data["developer"], SearchResultWeb)
return out
except Exception as err:
if hasattr(err, "response"):
diff --git a/apps/python-sdk/firecrawl/v2/methods/search.py b/apps/python-sdk/firecrawl/v2/methods/search.py
index 0f23ff00c..3b771bce1 100644
--- a/apps/python-sdk/firecrawl/v2/methods/search.py
+++ b/apps/python-sdk/firecrawl/v2/methods/search.py
@@ -42,8 +42,6 @@ def search(
out.news = _transform_array(data["news"], SearchResultNews)
if "images" in data:
out.images = _transform_array(data["images"], SearchResultImages)
- if "developer" in data:
- out.developer = _transform_array(data["developer"], SearchResultWeb)
return out
except Exception as err:
# If the error is an HTTP error from requests, handle it
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 1fe6bf4bc..4eeab4e9f 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -619,8 +619,8 @@ class Category(BaseModel):
website/domain filter and it returns ordinary web page results for those
domains β€” **not** paper records.
- "pdf": Filter results to PDF files (adds filetype:pdf to search)
- - "developer": Add developer results (issues, pull requests, READMEs and
- documentation) under `.developer`
+ - "developer": Developer-index results (issues, pull requests, READMEs and
+ documentation) served in `web`; cannot be combined with other categories
.. warning::
``categories=["research"]`` is **not** Firecrawl's research paper index.
@@ -1855,7 +1855,6 @@ class SearchData(BaseModel):
web: Optional[List[Union[SearchResultWeb, Document]]] = None
news: Optional[List[Union[SearchResultNews, Document]]] = None
images: Optional[List[Union[SearchResultImages, Document]]] = None
- developer: Optional[List[Union[SearchResultWeb, Document]]] = None
@property
def data(self):
@@ -1866,8 +1865,6 @@ class SearchData(BaseModel):
parts.append(f".news ({len(self.news)} results)")
if self.images:
parts.append(f".images ({len(self.images)} results)")
- if self.developer:
- parts.append(f".developer ({len(self.developer)} results)")
available = ", ".join(parts) if parts else ".web, .news, or .images"
raise AttributeError(
f"SearchData has no '.data'. Results are grouped by source: {available}"