Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
js-sdk: add dedicated developer search
Karan Lokchandani3185c6f61bad
diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex c0d9aa5c8..ae8d88c7a 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -198,6 +198,36 @@ const results = await app.search('nanopore basecalling accuracy', {> pubmed.ncbi.nlm.nih.gov, nature.com, biorxiv.org, ...) and returns web page> results. To search papers themselves, use `research.searchPapers` below.+### Developer search++Use `developerSearch` for the dedicated developer index and its complete filter+and response contract. Generic `search(..., { categories: ['developer'] })`+returns only the first passage as a description and does not accept these+filters.++```js+const evidence = await app.developerSearch('configure retry backoff', {+ repos: ['firecrawl/firecrawl'],+ types: ['issue', 'pull_request', 'readme'],+ passages: 3,+ language: 'TypeScript',+ license: 'MIT',+});++for (const result of evidence.results) {+ // Result kind is the id prefix; the API intentionally omits a type field.+ console.log(result.id, result.license);+ for (const passage of result.passages) {+ console.log(passage.text, passage.citation_url);+ }+}+console.log(evidence.repos); // indexed-status echoes for requested repos+```++`developerSearch` also supports `sources`, `topic`, `minStars`, `maxStars`,+`archived`, `fork`, and `skills: 'only'`. Supplying both `repos` and `sources`+OR-combines GitHub-backed and documentation results.+### Research / paper searchUse `app.research` to search Firecrawl's research paper index: ~43M paperdiff --git a/apps/js-sdk/firecrawl/src/__tests__/unit/v2/developer.test.ts b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/developer.test.tsnew file mode 100644index 000000000..98c6bef0d--- /dev/null+++ b/apps/js-sdk/firecrawl/src/__tests__/unit/v2/developer.test.ts@@ -0,0 +1,96 @@+import { describe, expect, jest, test } from "@jest/globals";+import { FirecrawlClient } from "../../../v2/client";++const response = {+ success: true,+ results: [+ {+ id: "issue:firecrawl/firecrawl#1",+ url: "https://github.com/firecrawl/firecrawl/issues/1",+ passages: [+ {+ text: "evidence",+ citation_url:+ "https://github.com/firecrawl/firecrawl/issues/1#issuecomment-1",+ },+ ],+ license: { state: "licensed" as const, spdx_id: "MIT" },+ },+ ],+ repos: [+ {+ repo: "firecrawl/firecrawl",+ indexed: true,+ types: { issue: true, pullRequest: true, readme: true },+ },+ ],+ sources: [{ source: "firecrawl", indexed: false }],+};++describe("developerSearch", () => {+ test("forwards every dedicated developer filter", async () => {+ const http = {+ post: jest.fn(async () => ({ status: 200, data: response })),+ } as any;++ const client = new FirecrawlClient({+ apiKey: "test",+ apiUrl: "http://localhost",+ });+ (client as any).http = http;++ const result = await client.developerSearch("retry configuration", {+ k: 20,+ passages: 5,+ types: ["doc", "issue", "pull_request", "readme"],+ repos: ["firecrawl/firecrawl"],+ sources: ["firecrawl"],+ language: "TypeScript",+ topic: ["web-scraping"],+ license: "MIT",+ minStars: 100,+ maxStars: 50000,+ archived: false,+ fork: true,+ skills: "only",+ });++ expect(http.post).toHaveBeenCalledWith("/v2/search/developer", {+ query: "retry configuration",+ k: 20,+ passages: 5,+ types: ["doc", "issue", "pull_request", "readme"],+ repos: ["firecrawl/firecrawl"],+ sources: ["firecrawl"],+ language: "TypeScript",+ topic: ["web-scraping"],+ license: "MIT",+ min_stars: 100,+ max_stars: 50000,+ archived: false,+ fork: true,+ skills: "only",+ });+ expect(result).toEqual(response);+ });++ test("accepts the flattened SPDX license response shape", async () => {+ const flattened = {+ ...response,+ results: [{ ...response.results[0], license: "MIT" }],+ };+ const http = {+ post: jest.fn(async () => ({ status: 200, data: flattened })),+ } as any;++ const client = new FirecrawlClient({+ apiKey: "test",+ apiUrl: "http://localhost",+ });+ (client as any).http = http;++ await expect(client.developerSearch("license shape")).resolves.toEqual(+ flattened,+ );+ });+});diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex f7035cc2b..ba6d6b245 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -6,6 +6,7 @@ import {} from "./methods/scrape";import { parse as parseMethod } from "./methods/parse";import { search } from "./methods/search";+import { developerSearch as developerSearchMethod } from "./methods/developer";import { map as mapMethod } from "./methods/map";import { feedback as feedbackMethod, searchFeedback as searchFeedbackMethod } from "./methods/feedback";import {@@ -51,6 +52,8 @@ import type {ScrapeOptions,SearchData,SearchRequest,+ DeveloperSearchOptions,+ DeveloperSearchResponse,EndpointFeedbackRequest,FeedbackResponse,SearchFeedbackRequest,@@ -240,6 +243,19 @@ export class FirecrawlClient {return search(this.http, { query, ...req });}+ /**+ * Search the dedicated developer index with repository, documentation,+ * artifact, language, topic, license, star, archive, and fork filters.+ * Unlike `search(query, { categories: ["developer"] })`, this returns all+ * requested passages, citations, license disclosures, and indexing echoes.+ */+ async developerSearch(+ query: string,+ options: DeveloperSearchOptions = {},+ ): Promise<DeveloperSearchResponse> {+ return developerSearchMethod(this.http, query, options);+ }+/*** Submit feedback for a v2 job.* @param request Feedback payload with endpoint, job id, rating, and supporting signals.diff --git a/apps/js-sdk/firecrawl/src/v2/methods/developer.ts b/apps/js-sdk/firecrawl/src/v2/methods/developer.tsnew file mode 100644index 000000000..d501c8573--- /dev/null+++ b/apps/js-sdk/firecrawl/src/v2/methods/developer.ts@@ -0,0 +1,36 @@+import type { DeveloperSearchOptions, DeveloperSearchResponse } from "../types";+import { HttpClient } from "../utils/httpClient";+import { throwForBadResponse } from "../utils/errorHandler";++const ENDPOINT = "/v2/search/developer";++/** Search the dedicated developer index with its complete filter surface. */+export async function developerSearch(+ http: HttpClient,+ query: string,+ options: DeveloperSearchOptions = {},+): Promise<DeveloperSearchResponse> {+ if (!query || !query.trim()) throw new Error("query cannot be empty");++ const response = await http.post<DeveloperSearchResponse>(ENDPOINT, {+ query,+ ...(options.k !== undefined ? { k: options.k } : {}),+ ...(options.passages !== undefined ? { passages: options.passages } : {}),+ ...(options.types !== undefined ? { types: options.types } : {}),+ ...(options.repos !== undefined ? { repos: options.repos } : {}),+ ...(options.sources !== undefined ? { sources: options.sources } : {}),+ ...(options.language !== undefined ? { language: options.language } : {}),+ ...(options.topic !== undefined ? { topic: options.topic } : {}),+ ...(options.license !== undefined ? { license: options.license } : {}),+ ...(options.minStars !== undefined ? { min_stars: options.minStars } : {}),+ ...(options.maxStars !== undefined ? { max_stars: options.maxStars } : {}),+ ...(options.archived !== undefined ? { archived: options.archived } : {}),+ ...(options.fork !== undefined ? { fork: options.fork } : {}),+ ...(options.skills !== undefined ? { skills: options.skills } : {}),+ });++ if (response.status !== 200) {+ throwForBadResponse(response, "search developer sources");+ }+ return response.data;+}diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex bf364606e..6241f6676 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -707,6 +707,76 @@ export interface PaginationConfig {maxWaitTime?: number;}+export type DeveloperSearchType = "doc" | "issue" | "pull_request" | "readme";++export interface DeveloperSearchOptions {+ /** Total ranked results, 1–100 (default 10). */+ k?: number;+ /** Passages per result, 1–5 (default 1). */+ passages?: number;+ /** Result kinds to search (default all). */+ types?: DeveloperSearchType[];+ /** GitHub owner/name filters, at most 20. */+ repos?: string[];+ /** Developer documentation source IDs, at most 20. */+ sources?: string[];+ /** One case-insensitive GitHub Linguist primary language. */+ language?: string;+ /** Repository topics that must all match, at most 8. */+ topic?: string[];+ /** One case-insensitive SPDX repository license identifier. */+ license?: string;+ /** Minimum repository stars. */+ minStars?: number;+ /** Maximum repository stars. */+ maxStars?: number;+ /** Filter by repository archived status. */+ archived?: boolean;+ /** Filter by repository fork status. */+ fork?: boolean;+ /** Return packaged agent-skill evidence only. */+ skills?: "only";+}++export interface DeveloperSearchLicenseDisclosure {+ state: "licensed" | "known_absent" | "unknown";+ spdx_id: string | null;+}++export interface DeveloperSearchPassage {+ text: string;+ citation_url?: string;+}++export interface DeveloperSearchResult {+ id: string;+ url: string;+ title?: string;+ passages: DeveloperSearchPassage[];+ // Accept both shapes while the API flattens license objects to SPDX strings.+ license?: DeveloperSearchLicenseDisclosure | string;+}++export interface DeveloperSearchRepoStatus {+ repo: string;+ indexed: boolean;+ types: { issue: boolean; pullRequest: boolean; readme: boolean };+}++export interface DeveloperSearchSourceStatus {+ source: string;+ indexed: boolean;+}++export interface DeveloperSearchResponse {+ success: boolean;+ results: DeveloperSearchResult[];+ /** Present only when repos were requested. */+ repos?: DeveloperSearchRepoStatus[];+ /** Present only when sources were requested. */+ sources?: DeveloperSearchSourceStatus[];+}+export interface SearchResultWeb {url: string;title?: string;