Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(sdks): research API accuracy, unified-client methods, origin telemetry (#4289)

Himadri Mishra8373dab922de
diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.md
index 982bf4348..c0d9aa5c8 100644
--- a/apps/js-sdk/firecrawl/README.md
+++ b/apps/js-sdk/firecrawl/README.md
@@ -166,6 +166,73 @@ const mapResult = await app.map('https://example.com');
console.log(mapResult);
```
+### Search
+
+Use `search` to search the web and optionally scrape the results in the same call.
+
+```js
+const results = await app.search('what is retrieval augmented generation?', { limit: 5 });
+for (const result of results.web ?? []) {
+ console.log(result.url, '-', result.title);
+}
+
+// Scrape every result as part of the search:
+const scraped = await app.search('firecrawl changelog', {
+ limit: 3,
+ scrapeOptions: { formats: ['markdown'] },
+});
+```
+
+Results are grouped by source: `.web`, `.news`, `.images` and `.developer`.
+
+Use `categories` to narrow web search to a kind of site:
+
+```js
+const results = await app.search('nanopore basecalling accuracy', {
+ categories: ['research'],
+});
+```
+
+> **`categories: ['research']` is a website filter, not the paper index.** It
+> restricts ordinary web search to roughly 14 academic domains (arxiv.org,
+> pubmed.ncbi.nlm.nih.gov, nature.com, biorxiv.org, ...) and returns web page
+> results. To search papers themselves, use `research.searchPapers` below.
+
+### Research / paper search
+
+Use `app.research` to search Firecrawl's research paper index: ~43M paper
+abstracts, roughly 90% biomedical and life sciences (PubMed, bioRxiv, medRxiv),
+plus arXiv for physics, mathematics and computer science.
+
+```js
+// Search the paper index (semantic search over abstracts):
+const papers = await app.research.searchPapers(
+ 'CRISPR base editing off-target effects in primary human T cells',
+ { k: 10 },
+);
+for (const paper of papers.results) {
+ console.log(paper.primaryId, '-', paper.title);
+}
+
+// Inspect one paper's metadata (accepts pmid:, pmcid:, doi: or arxiv: ids):
+const paper = await app.research.getPaper('pmid:<id>');
+
+// Read the passages inside a paper that answer a specific question:
+const read = await app.research.getPaper('pmid:<id>', {
+ query: 'what was the primary endpoint and the reported hazard ratio?',
+ k: 4,
+});
+
+// Expand along the citation graph, re-ranked for your stated intent:
+const related = await app.research.similarPapers('pmid:<id>', {
+ intent: 'replication attempts in larger cohorts',
+ k: 20,
+});
+```
+
+A companion `app.research.searchGithub` searches indexed GitHub issue/PR history
+and repository readmes.
+
### Scrape-bound interactive browsing (v2)
Use a scrape job ID to keep interacting with the replayed browser context:
diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.json
index b67b98ab5..ac1296803 100644
--- a/apps/js-sdk/firecrawl/package.json
+++ b/apps/js-sdk/firecrawl/package.json
@@ -1,7 +1,7 @@
{
"name": "@mendable/firecrawl-js",
- "version": "4.32.0",
- "description": "JavaScript SDK for Firecrawl API",
+ "version": "4.32.1",
+ "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
"main": "dist/index.js",
"types": "dist/index.d.ts",
"exports": {
@@ -57,7 +57,21 @@
"web",
"scraper",
"api",
- "sdk"
+ "sdk",
+ "web-search",
+ "literature-search",
+ "research",
+ "scientific-papers",
+ "academic-search",
+ "biomedical",
+ "life-sciences",
+ "pubmed",
+ "biorxiv",
+ "medrxiv",
+ "arxiv",
+ "preprints",
+ "citations",
+ "bioinformatics"
],
"engines": {
"node": ">=22.0.0"
diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.ts
index 926919657..e318eaf38 100644
--- a/apps/js-sdk/firecrawl/src/v1/index.ts
+++ b/apps/js-sdk/firecrawl/src/v1/index.ts
@@ -1642,7 +1642,7 @@ export default class FirecrawlApp {
* @param onActivity - Optional callback to receive activity updates in real-time.
* @param onSource - Optional callback to receive source updates in real-time.
* @returns The final research results.
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
*/
async deepResearch(
query: string,
@@ -1730,7 +1730,7 @@ export default class FirecrawlApp {
* Initiates a deep research operation on a given query without polling.
* @param params - Parameters for the deep research operation.
* @returns The response containing the research job ID.
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
*/
async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {
const headers = this.prepareHeaders();
@@ -1772,7 +1772,7 @@ export default class FirecrawlApp {
* Checks the status of a deep research operation.
* @param id - The ID of the deep research operation.
* @returns The current status and results of the research operation.
- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.
+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.
*/
async checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {
const headers = this.prepareHeaders();
diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.ts
index 42273692b..f7035cc2b 100644
--- a/apps/js-sdk/firecrawl/src/v2/client.ts
+++ b/apps/js-sdk/firecrawl/src/v2/client.ts
@@ -261,8 +261,19 @@ export class FirecrawlClient {
// Research
/**
- * Access the v2 research endpoints (arXiv papers + GitHub history/readmes).
- * Example: `firecrawl.research.searchPapers("diffusion models")`.
+ * Access the v2 research endpoints β€” Firecrawl's **research paper index**
+ * (~43M paper abstracts) plus GitHub history/readmes.
+ *
+ * The paper corpus is roughly 90% biomedical and life sciences β€” PubMed,
+ * bioRxiv and medRxiv β€” with arXiv covering physics, mathematics and
+ * computer science.
+ *
+ * ⚠️ Not the same as `search({ categories: ["research"] })`, which is only a
+ * website/domain filter over ordinary web search (~14 academic domains,
+ * returning page snippets). Use `research.searchPapers()` for literature
+ * search.
+ *
+ * Example: `firecrawl.research.searchPapers("GLP-1 receptor agonists cardiovascular outcomes")`.
*/
get research(): ResearchClient {
if (!this._research) this._research = new ResearchClient(this.http);
diff --git a/apps/js-sdk/firecrawl/src/v2/methods/research.ts b/apps/js-sdk/firecrawl/src/v2/methods/research.ts
index ffb9494ee..a05d32249 100644
--- a/apps/js-sdk/firecrawl/src/v2/methods/research.ts
+++ b/apps/js-sdk/firecrawl/src/v2/methods/research.ts
@@ -63,16 +63,47 @@ function normalizeResearchError(err: any, action: string): never {
}
/**
- * Client for the v2 research endpoints (arXiv papers + GitHub history/readmes).
+ * Client for the v2 research endpoints β€” Firecrawl's **research paper index**
+ * (~43M paper abstracts) plus GitHub history/readmes.
+ *
+ * The paper corpus is roughly 90% biomedical and life sciences β€” PubMed,
+ * bioRxiv and medRxiv β€” with arXiv covering physics, mathematics and computer
+ * science. Supports abstract search, paper metadata, in-body passage reads and
+ * citation-graph expansion.
+ *
+ * ⚠️ This is **not** the same as `search({ categories: ["research"] })`. That
+ * option is a website/domain filter on ordinary web search: it narrows
+ * Google-style results to ~14 academic domains and returns page snippets. The
+ * methods here query the paper index itself and return ranked paper records.
+ *
* Accessed via `firecrawl.research`.
+ *
+ * @example
+ * ```ts
+ * const res = await firecrawl.research.searchPapers(
+ * "tau aggregation inhibitors in Alzheimer's disease",
+ * { k: 10 },
+ * );
+ * ```
*/
export class ResearchClient {
constructor(private readonly http: HttpClient) {}
/**
- * Search papers by abstract relevance.
- * @param query Natural-language search query.
- * @param options Optional filters (k, authors, categories, from, to).
+ * Search the research paper index by abstract relevance.
+ *
+ * Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of the
+ * corpus β€” biomedical and life sciences) plus arXiv (physics, mathematics,
+ * computer science). Semantic search over abstracts, not keyword matching.
+ *
+ * This is **not** `search({ categories: ["research"] })`, which only narrows
+ * ordinary web search to ~14 academic websites.
+ *
+ * @param query Natural-language search query, e.g. `"CRISPR base editing
+ * off-target effects in primary human T cells"`.
+ * @param options Optional filters (k, authors, categories, from, to). Note
+ * `categories` here filters *paper* subject categories (e.g. `"q-bio.GN"`)
+ * and is unrelated to the `categories` option of `search()`.
*/
async searchPapers(
query: string,
@@ -102,8 +133,9 @@ export class ResearchClient {
/**
* Get paper metadata (detail mode), or read in-body passages (when `query` is
* supplied). `k` is only valid together with `query`.
- * @param id Paper reference: a canonical `paper_id`, an `arxiv:<id>` key, or a
- * bare arXiv id / URL.
+ * @param id Paper reference: a canonical `paperId`, or a namespaced id key
+ * such as `pmid:<id>`, `pmcid:<id>`, `doi:<doi>` or `arxiv:<id>`. Bare
+ * arXiv ids and arXiv URLs are also accepted.
* @param options Optional `query` (switches to read mode) and `k`.
*/
async getPaper(
diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.ts
index e1ec6c958..1f99c6232 100644
--- a/apps/js-sdk/firecrawl/src/v2/types.ts
+++ b/apps/js-sdk/firecrawl/src/v2/types.ts
@@ -694,6 +694,28 @@ export interface SearchData {
developer?: Array<SearchResultWeb | Document>;
}
+/**
+ * Narrows ordinary **web search**. A category does not switch `search()` to a
+ * different index.
+ *
+ * - `github` β€” restrict web results to github.com (a `site:` filter).
+ * - `research` β€” restrict web results to a fixed list of ~14 academic
+ * *websites* (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, science.org,
+ * ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns
+ * ordinary web page results from those domains, **not** paper records.
+ * - `pdf` β€” restrict results to PDFs (adds `filetype:pdf`).
+ * - `developer` β€” add developer results (issues, pull requests, READMEs and
+ * documentation) under `.developer`.
+ *
+ * ⚠️ `categories: ["research"]` is **not** Firecrawl's research paper index.
+ * To search papers themselves β€” ~43M abstracts, roughly 90% biomedical
+ * (PubMed, bioRxiv, medRxiv) plus arXiv β€” with full abstracts, in-body passage
+ * reads and citation-graph expansion, use `firecrawl.research.searchPapers()`
+ * (plus `getPaper()` and `similarPapers()`), which call `/v2/search/research`.
+ *
+ * Rule of thumb: literature search β†’ `research.searchPapers()`; web pages that
+ * happen to live on academic domains β†’ `search({ categories: ["research"] })`.
+ */
export interface CategoryOption {
type: "github" | "research" | "pdf" | "developer";
}
@@ -703,6 +725,14 @@ export interface SearchRequest {
sources?: Array<
"web" | "news" | "images" | { type: "web" | "news" | "images" }
>;
+ /**
+ * Narrow web search by category. See {@link CategoryOption}.
+ *
+ * ⚠️ `"research"` is a website/domain filter over ordinary web search (~14
+ * academic domains), **not** the research paper index. For literature search
+ * over ~43M paper abstracts (PubMed / bioRxiv / medRxiv / arXiv) use
+ * `firecrawl.research.searchPapers()` instead.
+ */
categories?: Array<
"github" | "research" | "pdf" | "developer" | CategoryOption
>;
@@ -1362,8 +1392,20 @@ export interface BrowserListResponse {
// ---------- Research (v2) ----------
/**
- * Source identifiers grouped by namespace. Currently only `arxiv` is
- * populated; each value is an array of ids in that namespace.
+ * Source identifiers grouped by namespace. Each key is a namespace and each
+ * value is an array of ids in that namespace.
+ *
+ * Which namespaces appear depends on the paper's source. Biomedical records
+ * (PubMed, bioRxiv, medRxiv) typically carry `pmid`, `pmcid` and/or `doi`;
+ * arXiv records carry `arxiv`. The set is open β€” treat this as a lookup by
+ * namespace rather than a fixed schema, and prefer {@link PaperResult.primaryId}
+ * when you just need one citable identifier.
+ *
+ * @example
+ * ```ts
+ * const doi = paper.ids?.doi?.[0];
+ * const pmid = paper.ids?.pmid?.[0];
+ * ```
*/
export type IdMap = Record<string, string[]>;
@@ -1379,7 +1421,7 @@ export interface PaperSignals {
seedOverlap: number;
}
-/** A ranked paper. `paperId` is canonical; arXiv lives in `ids`. */
+/** A ranked paper. `paperId` is canonical; source ids (pmid, pmcid, doi, arxiv, ...) live in `ids`. */
export interface PaperResult {
/** Canonical paper id β€” the Milvus INT64 primary key as a decimal string. */
paperId: string;
diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.md
index 5c2a2e461..ad571ee4d 100644
--- a/apps/python-sdk/README.md
+++ b/apps/python-sdk/README.md
@@ -180,6 +180,80 @@ map_result = firecrawl.map('https://firecrawl.dev')
print(map_result)
```
+### Search
+
+Use `search` to search the web and optionally scrape the results in the same call.
+
+```python
+# Search the web (v2):
+results = firecrawl.search("what is retrieval augmented generation?", limit=5)
+for result in results.web or []:
+ print(result.url, "-", result.title)
+
+# Scrape every result as part of the search:
+results = firecrawl.search(
+ "firecrawl changelog",
+ limit=3,
+ scrape_options={"formats": ["markdown"]},
+)
+```
+
+Results are grouped by source: `.web`, `.news`, `.images` and `.developer`.
+
+Use `categories` to narrow web search to a kind of site:
+
+```python
+results = firecrawl.search("nanopore basecalling accuracy", categories=["research"])
+```
+
+> **`categories=["research"]` is a website filter, not the paper index.** It
+> restricts ordinary web search to roughly 14 academic domains (arxiv.org,
+> pubmed.ncbi.nlm.nih.gov, nature.com, biorxiv.org, ...) and returns web page
+> results. To search papers themselves, use `search_papers` below.
+
+### Research / paper search
+
+Use `search_papers` to search Firecrawl's research paper index: ~43M paper
+abstracts, roughly 90% biomedical and life sciences (PubMed, bioRxiv, medRxiv),
+plus arXiv for physics, mathematics and computer science.
+
+```python
+# Search the paper index (semantic search over abstracts):
+papers = firecrawl.search_papers(
+ "CRISPR base editing off-target effects in primary human T cells",
+ k=10,
+)
+for paper in papers["results"]:
+ print(paper["primaryId"], "-", paper["title"])
+
+# Inspect one paper's metadata (accepts pmid:, pmcid:, doi: or arxiv: ids):
+paper = firecrawl.inspect_paper("pmid:<id>")
+
+# Read the passages inside a paper that answer a specific question:
+passages = firecrawl.read_paper(
+ "pmid:<id>",
+ "what was the primary endpoint and the reported hazard ratio?",
+ k=4,
+)
+
+# Expand along the citation graph, re-ranked for your stated intent:
+related = firecrawl.related_papers(
+ "pmid:<id>",
+ intent="replication attempts in larger cohorts",
+ k=20,
+)
+```
+
+A companion `search_github` searches indexed GitHub issue/PR history and repo
+readmes.
+
+> **Response keys are camelCase.** Unlike the rest of the SDK, the research
+> methods return the raw JSON body as a `dict` β€” they are not parsed into typed
+> models and not normalized to snake_case. Expect `paperId`, `primaryId`,
+> `createdDate`, `articleRank`, `poolSize`, and so on.
+
+Every method above is also available on `AsyncFirecrawl` with the same name.
+
### Scrape-bound interactive browsing (v2)
Use a scrape job ID to keep interacting with the replayed browser context:
diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.py
index 02bad5b9d..908e06d0d 100644
--- a/apps/python-sdk/firecrawl/__init__.py
+++ b/apps/python-sdk/firecrawl/__init__.py
@@ -18,7 +18,7 @@ from .v1 import (
V1ChangeTrackingOptions,
)
-__version__ = "4.34.0"
+__version__ = "4.35.0"
# Define the logger for the Firecrawl project
logger: logging.Logger = logging.getLogger("firecrawl")
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_research.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_research.py
new file mode 100644
index 000000000..2e23065e9
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_research.py
@@ -0,0 +1,286 @@
+"""
+Unit tests for the async v2 research paper index methods.
+
+Async mirror of ``firecrawl/__tests__/unit/v2/methods/test_research.py``: the
+async HTTP client is mocked and the tests pin the method names, the request
+paths and the query-string construction. No network access.
+"""
+
+from urllib.parse import parse_qs, urlparse
+
+import pytest
+
+from firecrawl.v2.methods.aio import research as aio_research
+from firecrawl.v2.utils.error_handler import BadRequestError, FirecrawlError
+
+
+class FakeResponse:
+ """Minimal stand-in for an httpx/requests-style response."""
+
+ def __init__(self, status_code=200, payload=None):
+ self.status_code = status_code
+ self._payload = {} if payload is None else payload
+ self.text = str(self._payload)
+
+ def json(self):
+ return self._payload
+
+
+class FakeAsyncHttpClient:
+ """Records every path requested through ``await get``."""
+
+ def __init__(self, response=None):
+ self.calls = []
+ self._response = response or FakeResponse()
+
+ async def get(self, path):
+ self.calls.append(path)
+ return self._response
+
+ @property
+ def last_path(self):
+ return self.calls[-1]
+
+
+def split(path):
+ """Split a recorded path into (path, {param: [values]})."""
+ parsed = urlparse(path)
+ return parsed.path, parse_qs(parsed.query, keep_blank_values=True)
+
+
+async def test_search_papers_builds_query_string_with_exploded_arrays():
+ client = FakeAsyncHttpClient(FakeResponse(200, {"success": True, "results": []}))
+ await aio_research.search_papers(
+ client,
+ "diffusion models",
+ k=10,
+ authors=["Ho", "Abbeel"],
+ categories=["cs.LG", "stat.ML"],
+ from_date="2020-01-01",
+ to_date="2024-12-31",
+ )
+ path, qs = split(client.last_path)
+ assert path == "/v2/search/research/papers"
+ assert qs["query"] == ["diffusion models"]
+ assert qs["k"] == ["10"]
+ assert qs["authors"] == ["Ho", "Abbeel"]
+ assert qs["categories"] == ["cs.LG", "stat.ML"]
+ assert qs["from"] == ["2020-01-01"]
+ assert qs["to"] == ["2024-12-31"]
+
+
+async def test_search_papers_omits_absent_options():
+ client = FakeAsyncHttpClient(FakeResponse(200, {"success": True, "results": []}))
+ await aio_research.search_papers(client, "q")
+ path, qs = split(client.last_path)
+ assert path == "/v2/search/research/papers"
+ assert sorted(qs.keys()) == ["origin", "query"]
+ assert qs["origin"][0].startswith("python-sdk@")
+
+
+async def test_search_papers_returns_body_verbatim_in_camel_case():
+ payload = {
+ "success": True,
+ "results": [{"paperId": "1", "primaryId": "pmid:12345678", "score": 0.5}],
+ }
+ client = FakeAsyncHttpClient(FakeResponse(200, payload))
+ result = await aio_research.search_papers(client, "q")
+ assert result == payload
+ assert "paperId" in result["results"][0]
+ assert "paper_id" not in result["results"][0]
+
+
+ "paper_id, expected_path",
+ [
+ ("arxiv:2105.05233", "/v2/search/research/papers/arxiv%3A2105.05233"),
+ ("pmid:12345678", "/v2/search/research/papers/pmid%3A12345678"),
+ ("pmcid:PMC1234567", "/v2/search/research/papers/pmcid%3APMC1234567"),
+ (
+ "doi:10.1038/s41586-020-2649-2",
+ "/v2/search/research/papers/doi%3A10.1038%2Fs41586-020-2649-2",
+ ),
+ ],
+)
+async def test_inspect_paper_encodes_namespaced_ids(paper_id, expected_path):
+ client = FakeAsyncHttpClient(FakeResponse(200, {"success": True, "paper": {}}))
+ await aio_research.inspect_paper(client, paper_id)
+ path, qs = split(client.last_path)
+ assert path == expected_path
+ # detail mode carries origin, like the other four research methods
+ assert sorted(qs.keys()) == ["origin"]
+ assert qs["origin"][0].startswith("python-sdk@")
+
+
+async def test_read_paper_adds_query_and_k():
+ client = FakeAsyncHttpClient(
+ FakeResponse(200, {"success": True, "paper": {}, "passages": []})
+ )
+ await aio_research.read_paper(client, "pmid:12345678", "primary endpoint", k=4)
+ path, qs = split(client.last_path)
+ assert path == "/v2/search/research/papers/pmid%3A12345678"
+ assert qs["query"] == ["primary endpoint"]
+ assert qs["k"] == ["4"]
+
+
+async def test_related_papers_builds_path_and_query():
+ client = FakeAsyncHttpClient(
+ FakeResponse(200, {"success": True, "results": [], "poolSize": 0, "truncated": False})
+ )
+ await aio_research.related_papers(
+ client,
+ "2105.05233",
+ "diffusion image synthesis",
+ mode="citers",
+ k=20,
+ rerank=False,
+ anchor=["arxiv:2006.11239", "1503.03585"],
+ )
+ path, qs = split(client.last_path)
+ assert path == "/v2/search/research/papers/2105.05233/similar"
+ assert qs["intent"] == ["diffusion image synthesis"]
+ assert qs["mode"] == ["citers"]
+ assert qs["k"] == ["20"]
+ assert qs["rerank"] == ["false"]
+ assert qs["anchor"] == ["arxiv:2006.11239", "1503.03585"]
+
+
+async def test_related_papers_omits_rerank_when_none():
+ client = FakeAsyncHttpClient(FakeResponse(200, {"success": True, "results": []}))
+ await aio_research.related_papers(client, "1", "intent")
+ _, qs = split(client.last_path)
+ assert "rerank" not in qs
+
+
+async def test_search_github_builds_query_string():
+ client = FakeAsyncHttpClient(FakeResponse(200, {"success": True, "results": []}))
+ await aio_research.search_github(client, "milvus hybrid search", k=10)
+ path, qs = split(client.last_path)
+ assert path == "/v2/search/research/github"
+ assert qs["query"] == ["milvus hybrid search"]
+ assert qs["k"] == ["10"]
+
+
+async def test_error_mapping():
+ client = FakeAsyncHttpClient(FakeResponse(400, {"error": "query is required"}))
+ with pytest.raises(BadRequestError) as exc:
+ await aio_research.search_papers(client, "q")
+ assert "query is required" in str(exc.value)
+ assert exc.value.status_code == 400
+
+ client = FakeAsyncHttpClient(FakeResponse(404, {"error": "Not Found"}))
+ with pytest.raises(FirecrawlError) as exc:
+ await aio_research.inspect_paper(client, "pmid:0")
+ assert exc.value.status_code == 404
+
+
+async def test_async_client_exposes_research_methods():
+ from firecrawl.v2.client_async import AsyncFirecrawlClient
+
+ for name in (
+ "search_papers",
+ "inspect_paper",
+ "read_paper",
+ "related_papers",
+ "search_github",
+ ):
+ assert callable(getattr(AsyncFirecrawlClient, name)), name
+
+ client = AsyncFirecrawlClient(api_key="fc-test")
+ client.async_http_client = FakeAsyncHttpClient(
+ FakeResponse(200, {"success": True, "results": []})
+ )
+ await client.search_papers("tau aggregation inhibitors", k=5)
+ path, qs = split(client.async_http_client.last_path)
+ assert path == "/v2/search/research/papers"
+ assert qs["k"] == ["5"]
+
+
+async def test_async_research_methods_are_documented():
+ """The docstrings are the disambiguation surface agents actually read."""
+ from firecrawl.v2.client_async import AsyncFirecrawlClient
+
+ doc = AsyncFirecrawlClient.search_papers.__doc__ or ""
+ assert "PubMed" in doc
+ assert 'categories=["research"]' in doc
+
+
+RESEARCH_METHODS = (
+ "search_papers",
+ "inspect_paper",
+ "read_paper",
+ "related_papers",
+ "search_github",
+)
+
+
+def _unified_client(response=None):
+ """Top-level ``AsyncFirecrawl`` with its v2 HTTP layer mocked."""
+ from firecrawl import AsyncFirecrawl
+
+ client = AsyncFirecrawl(api_key="fc-test")
+ client._v2_client.async_http_client = FakeAsyncHttpClient(
+ response or FakeResponse(200, {"success": True})
+ )
+ return client
+
+
+def test_unified_async_client_exposes_research_methods():
+ """``AsyncFirecrawl`` delegates method by method; pin all five."""
+ from firecrawl import AsyncFirecrawl
+
+ for name in RESEARCH_METHODS:
+ assert callable(getattr(AsyncFirecrawl, name, None)), name
+ assert (getattr(AsyncFirecrawl, name).__doc__ or "").strip(), name
+
+ doc = AsyncFirecrawl.search_papers.__doc__ or ""
+ assert "PubMed" in doc
+ assert 'categories=["research"]' in doc
+
+
+async def test_unified_async_client_delegates_all_research_methods():
+ client = _unified_client(FakeResponse(200, {"success": True, "results": []}))
+ http = client._v2_client.async_http_client
+
+ await client.search_papers("tau aggregation inhibitors", k=5)
+ path, qs = split(http.last_path)
+ assert path == "/v2/search/research/papers"
+ assert qs["k"] == ["5"]
+
+ await client.inspect_paper("pmid:12345678")
+ path, _ = split(http.last_path)
+ assert path == "/v2/search/research/papers/pmid%3A12345678"
+
+ await client.read_paper("pmid:12345678", "primary endpoint", k=4)
+ path, qs = split(http.last_path)
+ assert path == "/v2/search/research/papers/pmid%3A12345678"
+ assert qs["query"] == ["primary endpoint"]
+
+ await client.related_papers("pmid:12345678", intent="replication attempts")
+ path, qs = split(http.last_path)
+ assert path == "/v2/search/research/papers/pmid%3A12345678/similar"
+ assert qs["intent"] == ["replication attempts"]
+
+ await client.search_github("pysam VCF parsing memory leak")
+ path, qs = split(http.last_path)
+ assert path == "/v2/search/research/github"
+ assert qs["query"] == ["pysam VCF parsing memory leak"]
+
+
+async def test_unified_async_client_returns_response_verbatim():
+ payload = {"success": True, "results": [{"paperId": "1", "primaryId": "pmid:1"}]}
+ client = _unified_client(FakeResponse(200, payload))
+ assert await client.search_papers("q") == payload
diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_research.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_research.py
new file mode 100644
index 000000000..322a1c7ba
--- /dev/null
+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_research.py
@@ -0,0 +1,434 @@
+"""
+Unit tests for the v2 research paper index methods.
+
+Mirrors the JS SDK suite in
+``apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts``: the HTTP
+client is mocked and the tests pin the method names, the request paths and the
+query-string construction. No network access.
+
+These endpoints are the research paper index (``/v2/search/research``) β€” not the
+``search(categories=["research"])`` website filter.
+"""
+
+from urllib.parse import parse_qs, urlparse
+
+import pytest
+
+from firecrawl.v2.methods import research
+from firecrawl.v2.utils.error_handler import (
+ BadRequestError,
+ FirecrawlError,
+)
+
+
+class FakeResponse:
+ """Minimal stand-in for ``requests.Response``."""
+
+ def __init__(self, status_code=200, payload=None):
+ self.status_code = status_code
+ self._payload = {} if payload is None else payload
+ self.text = str(self._payload)
+
+ def json(self):
+ return self._payload
+
+
+class FakeHttpClient:
+ """Records every path requested through ``get``."""
+
+ def __init__(self, response=None):
+ self.calls = []
+ self._response = response or FakeResponse()
+
+ def get(self, path):
+ self.calls.append(path)
+ return self._response
+
+ @property
+ def last_path(self):
+ return self.calls[-1]
+
+
+def split(path):
+ """Split a recorded path into (path, {param: [values]})."""
+ parsed = urlparse(path)
+ return parsed.path, parse_qs(parsed.query, keep_blank_values=True)
+
+
+class TestSearchPapers:
+ """search_papers -> GET /v2/search/research/papers"""
+
+ def test_builds_query_string_with_exploded_arrays(self):
+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))
+ research.search_papers(
+ client,
+ "diffusion models",
+ k=10,
+ authors=["Ho", "Abbeel"],
+ categories=["cs.LG", "stat.ML"],
+ from_date="2020-01-01",
+ to_date="2024-12-31",
+ )
+ path, qs = split(client.last_path)
+ assert path == "/v2/search/research/papers"
+ assert qs["query"] == ["diffusion models"]
+ assert qs["k"] == ["10"]
+ # list values are exploded into repeated params, not comma-joined
+ assert qs["authors"] == ["Ho", "Abbeel"]
+ assert qs["categories"] == ["cs.LG", "stat.ML"]
+ # from_date/to_date are sent as the wire names `from`/`to`
+ assert qs["from"] == ["2020-01-01"]
+ assert qs["to"] == ["2024-12-31"]
+
+ def test_biomedical_query_is_passed_through_percent_encoded(self):
+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))
+ research.search_papers(
+ client, "CRISPR base editing off-target effects in T cells"
+ )
+ assert "CRISPR%20base%20editing" in client.last_path
+ _, qs = split(client.last_path)
+ assert qs["query"] == ["CRISPR base editing off-target effects in T cells"]
+
+ def test_omits_absent_options(self):
+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))
+ research.search_papers(client, "q")
+ path, qs = split(client.last_path)
+ assert path == "/v2/search/research/papers"
+ assert sorted(qs.keys()) == ["origin", "query"]
+ assert qs["origin"][0].startswith("python-sdk@")
+
+ def test_returns_the_response_body_verbatim(self):
+ payload = {
+ "success": True,
+ "results": [
+ {
+ "paperId": "1",
+ "primaryId": "pmid:12345678",
+ "title": "t",
+ "abstract": "a",
+ "score": 0.1,
+ }
+ ],
+ }
+ client = FakeHttpClient(FakeResponse(200, payload))
+ assert research.search_papers(client, "q") == payload
+
+ def test_response_keys_are_not_snake_case_normalized(self):
+ """Research responses are returned raw; keys stay camelCase."""
+ payload = {"success": True, "results": [{"paperId": "1", "primaryId": "doi:10.1/x"}]}
+ client = FakeHttpClient(FakeResponse(200, payload))
+ result = research.search_papers(client, "q")
+ assert "paperId" in result["results"][0]
+ assert "paper_id" not in result["results"][0]
+
+
+class TestInspectPaper:
+ """inspect_paper -> GET /v2/search/research/papers/{id}"""
+
+ @pytest.mark.parametrize(
+ "paper_id, expected_path",
+ [
+ ("arxiv:2105.05233", "/v2/search/research/papers/arxiv%3A2105.05233"),
+ ("pmid:12345678", "/v2/search/research/papers/pmid%3A12345678"),
+ ("pmcid:PMC1234567", "/v2/search/research/papers/pmcid%3APMC1234567"),
+ (
+ "doi:10.1038/s41586-020-2649-2",
+ "/v2/search/research/papers/doi%3A10.1038%2Fs41586-020-2649-2",
+ ),
+ ("123456789", "/v2/search/research/papers/123456789"),
+ ],
+ )
+ def test_encodes_namespaced_ids_in_the_path(self, paper_id, expected_path):
+ client = FakeHttpClient(FakeResponse(200, {"success": True, "paper": {}}))
+ research.inspect_paper(client, paper_id)
+ path, _ = split(client.last_path)
+ assert path == expected_path
+
+ def test_sends_origin_like_the_other_research_methods(self):
+ """Detail mode sends ``origin`` too β€” same attribution as the other four."""
+ client = FakeHttpClient(FakeResponse(200, {"success": True, "paper": {}}))
+ research.inspect_paper(client, "pmid:12345678")
+ _, qs = split(client.last_path)
+ assert sorted(qs.keys()) == ["origin"]
+ assert qs["origin"][0].startswith("python-sdk@")
+
+
+class TestReadPaper:
+ """read_paper -> GET /v2/search/research/papers/{id}?query=..."""
+
+ def test_adds_query_and_k(self):
+ client = FakeHttpClient(
+ FakeResponse(200, {"success": True, "paper": {}, "passages": []})
+ )
+ research.read_paper(
+ client,
+ "pmid:12345678",
+ "what was the primary endpoint?",
+ k=4,
+ )
+ path, qs = split(client.last_path)
+ assert path == "/v2/search/research/papers/pmid%3A12345678"
+ assert qs["query"] == ["what was the primary endpoint?"]
+ assert qs["k"] == ["4"]
+ assert qs["origin"][0].startswith("python-sdk@")
+
+ def test_omits_k_when_absent(self):
+ client = FakeHttpClient(FakeResponse(200, {"success": True, "paper": {}}))
+ research.read_paper(client, "123", "noise schedule")
+ _, qs = split(client.last_path)
+ assert sorted(qs.keys()) == ["origin", "query"]
+
+
+class TestRelatedPapers:
+ """related_papers -> GET /v2/search/research/papers/{id}/similar
+
+ Same endpoint the JS SDK exposes as ``research.similarPapers()``.
+ """
+
+ def test_builds_path_and_query_with_repeated_anchors_and_rerank(self):
+ client = FakeHttpClient(
+ FakeResponse(200, {"success": True, "results": [], "poolSize": 0, "truncated": False})
+ )
+ research.related_papers(
+ client,
+ "2105.05233",
+ "diffusion image synthesis",
+ mode="citers",
+ k=20,
+ rerank=False,
+ anchor=["arxiv:2006.11239", "1503.03585"],
+ )
+ path, qs = split(client.last_path)
+ assert path == "/v2/search/research/papers/2105.05233/similar"
+ assert qs["intent"] == ["diffusion image synthesis"]
+ assert qs["mode"] == ["citers"]
+ assert qs["k"] == ["20"]
+ # booleans are lower-cased, not Python-cased
+ assert qs["rerank"] == ["false"]
+ assert qs["anchor"] == ["arxiv:2006.11239", "1503.03585"]
+
+ def test_rerank_true_is_lowercased(self):
+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))
+ research.related_papers(client, "1", "intent", rerank=True)
+ _, qs = split(client.last_path)
+ assert qs["rerank"] == ["true"]
+
+ def test_omits_rerank_when_none(self):
+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))
+ research.related_papers(client, "1", "intent")
+ _, qs = split(client.last_path)
+ assert "rerank" not in qs
+ assert qs["intent"] == ["intent"]
+
+
+class TestSearchGithub:
+ """search_github -> GET /v2/search/research/github"""
+
+ def test_builds_query_string(self):
+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))
+ research.search_github(client, "milvus hybrid search", k=10)
+ path, qs = split(client.last_path)
+ assert path == "/v2/search/research/github"
+ assert qs["query"] == ["milvus hybrid search"]
+ assert qs["k"] == ["10"]
+ assert qs["origin"][0].startswith("python-sdk@")
+
+
+class TestErrorMapping:
+ """Non-200 responses raise the shared v2 error types."""
+
+ def test_400_raises_bad_request(self):
+ client = FakeHttpClient(FakeResponse(400, {"error": "query is required"}))
+ with pytest.raises(BadRequestError) as exc:
+ research.search_papers(client, "q")
+ assert "query is required" in str(exc.value)
+ assert exc.value.status_code == 400
+
+ def test_404_raises_firecrawl_error(self):
+ client = FakeHttpClient(FakeResponse(404, {"error": "Not Found"}))
+ with pytest.raises(FirecrawlError) as exc:
+ research.inspect_paper(client, "pmid:0")
+ assert exc.value.status_code == 404
+
+ def test_error_action_is_labelled_research(self):
+ client = FakeHttpClient(FakeResponse(500, {"error": "boom"}))
+ with pytest.raises(FirecrawlError) as exc:
+ research.search_github(client, "q")
+ assert "research" in str(exc.value)
+
+
+class TestClientWiring:
+ """Pin the public method names on the sync client."""
+
+ def _client(self, response=None):
+ from firecrawl.v2.client import FirecrawlClient
+
+ client = FirecrawlClient(api_key="fc-test")
+ client.http_client = FakeHttpClient(response or FakeResponse(200, {"success": True}))
+ return client
+
+ def test_client_exposes_research_methods(self):
+ from firecrawl.v2.client import FirecrawlClient
+
+ for name in (
+ "search_papers",
+ "inspect_paper",
+ "read_paper",
+ "related_papers",
+ "search_github",
+ ):
+ assert callable(getattr(FirecrawlClient, name)), name
+
+ def test_search_papers_routes_to_the_paper_index(self):
+ client = self._client(FakeResponse(200, {"success": True, "results": []}))
+ client.search_papers("tau aggregation inhibitors", k=5)
+ path, qs = split(client.http_client.last_path)
+ assert path == "/v2/search/research/papers"
+ assert qs["k"] == ["5"]
+
+ def test_read_paper_routes_to_the_paper_id_path(self):
+ client = self._client(FakeResponse(200, {"success": True, "paper": {}}))
+ client.read_paper("pmid:12345678", "primary endpoint", k=2)
+ path, qs = split(client.http_client.last_path)
+ assert path == "/v2/search/research/papers/pmid%3A12345678"
+ assert qs["query"] == ["primary endpoint"]
+
+ def test_related_papers_routes_to_similar(self):
+ client = self._client(FakeResponse(200, {"success": True, "results": []}))
+ client.related_papers("pmid:12345678", intent="replication attempts")
+ path, _ = split(client.http_client.last_path)
+ assert path == "/v2/search/research/papers/pmid%3A12345678/similar"
+
+ def test_research_methods_are_documented(self):
+ """The docstrings are the disambiguation surface agents actually read."""
+ from firecrawl.v2.client import FirecrawlClient
+
+ doc = FirecrawlClient.search_papers.__doc__ or ""
+ assert "PubMed" in doc
+ assert 'categories=["research"]' in doc
+
+
+RESEARCH_METHODS = (
+ "search_papers",
+ "inspect_paper",
+ "read_paper",
+ "related_papers",
+ "search_github",
+)
+
+
+class TestUnifiedClientWiring:
+ """The top-level ``Firecrawl`` client is what the README instantiates.
+
+ ``Firecrawl`` does not inherit from the v2 client (unlike the JS SDK, where
+ ``Firecrawl extends FirecrawlClient``) β€” it delegates method by method. Any
+ method missing from that list is simply absent from the documented surface,
+ so pin all five here.
+ """
+
+ def _client(self, response=None):
+ from firecrawl import Firecrawl
+
+ client = Firecrawl(api_key="fc-test")
+ client._v2_client.http_client = FakeHttpClient(
+ response or FakeResponse(200, {"success": True})
+ )
+ return client
+
+ def test_unified_client_exposes_research_methods(self):
+ from firecrawl import Firecrawl
+
+ for name in RESEARCH_METHODS:
+ assert callable(getattr(Firecrawl, name, None)), name
+
+ def test_search_papers_delegates_to_the_paper_index(self):
+ client = self._client(FakeResponse(200, {"success": True, "results": []}))
+ client.search_papers("tau aggregation inhibitors", k=5)
+ path, qs = split(client._v2_client.http_client.last_path)
+ assert path == "/v2/search/research/papers"
+ assert qs["query"] == ["tau aggregation inhibitors"]
+ assert qs["k"] == ["5"]
+
+ def test_inspect_paper_delegates(self):
+ client = self._client(FakeResponse(200, {"success": True, "paper": {}}))
+ client.inspect_paper("pmid:12345678")
+ path, _ = split(client._v2_client.http_client.last_path)
+ assert path == "/v2/search/research/papers/pmid%3A12345678"
+
+ def test_read_paper_delegates(self):
+ client = self._client(FakeResponse(200, {"success": True, "passages": []}))
+ client.read_paper("pmid:12345678", "primary endpoint", k=4)
+ path, qs = split(client._v2_client.http_client.last_path)
+ assert path == "/v2/search/research/papers/pmid%3A12345678"
+ assert qs["query"] == ["primary endpoint"]
+ assert qs["k"] == ["4"]
+
+ def test_related_papers_delegates(self):
+ client = self._client(FakeResponse(200, {"success": True, "results": []}))
+ client.related_papers("pmid:12345678", intent="replication attempts", k=20)
+ path, qs = split(client._v2_client.http_client.last_path)
+ assert path == "/v2/search/research/papers/pmid%3A12345678/similar"
+ assert qs["intent"] == ["replication attempts"]
+ assert qs["k"] == ["20"]
+
+ def test_search_github_delegates(self):
+ client = self._client(FakeResponse(200, {"success": True, "results": []}))
+ client.search_github("pysam VCF parsing memory leak", k=3)
+ path, qs = split(client._v2_client.http_client.last_path)
+ assert path == "/v2/search/research/github"
+ assert qs["query"] == ["pysam VCF parsing memory leak"]
+
+ def test_returns_the_v2_response_verbatim(self):
+ payload = {"success": True, "results": [{"paperId": "1", "primaryId": "pmid:1"}]}
+ client = self._client(FakeResponse(200, payload))
+ assert client.search_papers("q") == payload
+
+ def test_delegated_docstrings_carry_the_disambiguation(self):
+ from firecrawl import Firecrawl
+
+ doc = Firecrawl.search_papers.__doc__ or ""
+ assert "PubMed" in doc
+ assert 'categories=["research"]' in doc
+ for name in RESEARCH_METHODS:
+ assert (getattr(Firecrawl, name).__doc__ or "").strip(), name
+
+
+class TestDocstringSingleSourceOfTruth:
+ """The five docstrings live once, in ``v2/methods/research_docs.py``.
+
+ Six layers surface the same five methods. Each keeps a full docstring, but
+ the prose is applied from the canonical constants, so a correction to a
+ factual claim is a one-place edit. These tests fail if a layer forks its
+ own copy again.
+ """
+
+ def _layers(self):
+ from firecrawl import AsyncFirecrawl, Firecrawl
+ from firecrawl.v2.client import FirecrawlClient
+ from firecrawl.v2.client_async import AsyncFirecrawlClient
+ from firecrawl.v2.methods import research as sync_research
+ from firecrawl.v2.methods import research_docs as docs
+ from firecrawl.v2.methods.aio import research as aio_research
+
+ return docs, {
+ "impl": (sync_research, ""),
+ "impl_aio": (aio_research, "AIO_"),
+ "v2_client": (FirecrawlClient, "CLIENT_"),
+ "v2_client_async": (AsyncFirecrawlClient, "ASYNC_CLIENT_"),
+ "unified": (Firecrawl, "CLIENT_"),
+ "unified_async": (AsyncFirecrawl, "ASYNC_CLIENT_"),
+ }
+
+ def test_every_layer_uses_the_canonical_docstring(self):
+ docs, layers = self._layers()
+ for name in RESEARCH_METHODS:
+ for label, (obj, prefix) in layers.items():
+ canonical = getattr(docs, f"{prefix}{name.upper()}_DOC")
+ assert getattr(obj, name).__doc__ == canonical, f"{label}.{name}"
+
+ def test_no_unresolved_placeholders_ship_to_users(self):
+ """A template placeholder leaking into ``help()`` would be user-visible."""
+ docs, layers = self._layers()
+ for name in RESEARCH_METHODS:
+ for obj, _ in layers.values():
+ assert "|" not in (getattr(obj, name).__doc__ or "")
diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.py
index c998d3869..eb148aed9 100644
--- a/apps/python-sdk/firecrawl/client.py
+++ b/apps/python-sdk/firecrawl/client.py
@@ -26,6 +26,19 @@ import logging
from .v1 import V1FirecrawlApp, AsyncV1FirecrawlApp
from .v2 import FirecrawlClient as V2FirecrawlClient
from .v2.client_async import AsyncFirecrawlClient
+from .v2.methods.research_docs import (
+ ASYNC_CLIENT_INSPECT_PAPER_DOC,
+ ASYNC_CLIENT_READ_PAPER_DOC,
+ ASYNC_CLIENT_RELATED_PAPERS_DOC,
+ ASYNC_CLIENT_SEARCH_GITHUB_DOC,
+ ASYNC_CLIENT_SEARCH_PAPERS_DOC,
+ CLIENT_INSPECT_PAPER_DOC,
+ CLIENT_READ_PAPER_DOC,
+ CLIENT_RELATED_PAPERS_DOC,
+ CLIENT_SEARCH_GITHUB_DOC,
+ CLIENT_SEARCH_PAPERS_DOC,
+ doc,
+)
from .v2.types import Document, ParseOptions, ScrapeOptions
logger = logging.getLogger("firecrawl")
@@ -331,7 +344,29 @@ class Firecrawl:
content_type=content_type,
options=options,
)
-
+
+ # Research paper index (/v2/search/research) β€” delegates to the v2 client.
+ @doc(CLIENT_SEARCH_PAPERS_DOC)
+ def search_papers(self, query: str, **kwargs):
+ return self._v2_client.search_papers(query, **kwargs)
+
+ @doc(CLIENT_INSPECT_PAPER_DOC)
+ def inspect_paper(self, paper_id: str):
+ return self._v2_client.inspect_paper(paper_id)
+
+ @doc(CLIENT_READ_PAPER_DOC)
+ def read_paper(self, paper_id: str, query: str, **kwargs):
+ return self._v2_client.read_paper(paper_id, query, **kwargs)
+
+ @doc(CLIENT_RELATED_PAPERS_DOC)
+ def related_papers(self, paper_id: str, intent: str, **kwargs):
+ return self._v2_client.related_papers(paper_id, intent, **kwargs)
+
+ @doc(CLIENT_SEARCH_GITHUB_DOC)
+ def search_github(self, query: str, **kwargs):
+ return self._v2_client.search_github(query, **kwargs)
+
+
class AsyncFirecrawl:
"""Async unified Firecrawl client (v2 by default, v1 under ``.v1``)."""
@@ -444,6 +479,28 @@ class AsyncFirecrawl:
options=options,
)
+ # Research paper index (/v2/search/research) β€” delegates to the v2 client.
+ @doc(ASYNC_CLIENT_SEARCH_PAPERS_DOC)
+ async def search_papers(self, query: str, **kwargs):
+ return await self._v2_client.search_papers(query, **kwargs)
+
+ @doc(ASYNC_CLIENT_INSPECT_PAPER_DOC)
+ async def inspect_paper(self, paper_id: str):
+ return await self._v2_client.inspect_paper(paper_id)
+
+ @doc(ASYNC_CLIENT_READ_PAPER_DOC)
+ async def read_paper(self, paper_id: str, query: str, **kwargs):
+ return await self._v2_client.read_paper(paper_id, query, **kwargs)
+
+ @doc(ASYNC_CLIENT_RELATED_PAPERS_DOC)
+ async def related_papers(self, paper_id: str, intent: str, **kwargs):
+ return await self._v2_client.related_papers(paper_id, intent, **kwargs)
+
+ @doc(ASYNC_CLIENT_SEARCH_GITHUB_DOC)
+ async def search_github(self, query: str, **kwargs):
+ return await self._v2_client.search_github(query, **kwargs)
+
+
# Export Firecrawl as an alias for FirecrawlApp
FirecrawlApp = Firecrawl
AsyncFirecrawlApp = AsyncFirecrawl
diff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.py
index df4236af7..b6525ee0d 100644
--- a/apps/python-sdk/firecrawl/v1/client.py
+++ b/apps/python-sdk/firecrawl/v1/client.py
@@ -2631,7 +2631,7 @@ class V1FirecrawlApp:
Initiates a deep research operation on a given query and polls until completion.
.. deprecated::
- /v1/deep-research is deprecated. Use /v2/search instead.
+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.
Args:
query (str): Research query or topic to investigate
@@ -2660,7 +2660,7 @@ class V1FirecrawlApp:
"""
import warnings
warnings.warn(
- "/v1/deep-research is deprecated. Use /v2/search instead.",
+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",
DeprecationWarning,
stacklevel=2,
)
@@ -2734,7 +2734,7 @@ class V1FirecrawlApp:
Initiates an asynchronous deep research operation.
.. deprecated::
- /v1/deep-research is deprecated. Use /v2/search instead.
+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.
Args:
query (str): Research query or topic to investigate
@@ -2756,7 +2756,7 @@ class V1FirecrawlApp:
"""
import warnings
warnings.warn(
- "/v1/deep-research is deprecated. Use /v2/search instead.",
+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",
DeprecationWarning,
stacklevel=2,
)
@@ -2805,7 +2805,7 @@ class V1FirecrawlApp:
Check the status of a deep research operation.
.. deprecated::
- /v1/deep-research is deprecated. Use /v2/search instead.
+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.
Args:
id (str): The ID of the deep research operation.
@@ -2830,7 +2830,7 @@ class V1FirecrawlApp:
"""
import warnings
warnings.warn(
- "/v1/deep-research is deprecated. Use /v2/search instead.",
+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",
DeprecationWarning,
stacklevel=2,
)
@@ -4994,7 +4994,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
Initiates a deep research operation on a given query and polls until completion.
.. deprecated::
- /v1/deep-research is deprecated. Use /v2/search instead.
+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.
Args:
query (str): Research query or topic to investigate
@@ -5023,7 +5023,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
"""
import warnings
warnings.warn(
- "/v1/deep-research is deprecated. Use /v2/search instead.",
+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",
DeprecationWarning,
stacklevel=2,
)
@@ -5097,7 +5097,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
Initiates an asynchronous deep research operation.
.. deprecated::
- /v1/deep-research is deprecated. Use /v2/search instead.
+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.
Args:
query (str): Research query or topic to investigate
@@ -5119,7 +5119,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
"""
import warnings
warnings.warn(
- "/v1/deep-research is deprecated. Use /v2/search instead.",
+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",
DeprecationWarning,
stacklevel=2,
)
@@ -5157,7 +5157,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
Check the status of a deep research operation.
.. deprecated::
- /v1/deep-research is deprecated. Use /v2/search instead.
+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.
Args:
id (str): The ID of the deep research operation.
@@ -5182,7 +5182,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):
"""
import warnings
warnings.warn(
- "/v1/deep-research is deprecated. Use /v2/search instead.",
+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",
DeprecationWarning,
stacklevel=2,
)
diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.py
index 64371d85b..e2cbf8094 100644
--- a/apps/python-sdk/firecrawl/v2/client.py
+++ b/apps/python-sdk/firecrawl/v2/client.py
@@ -68,6 +68,14 @@ from .methods import agent as agent_module
from .methods import browser as browser_module
from .methods import monitor as monitor_module
from .methods import research as research_module
+from .methods.research_docs import (
+ CLIENT_INSPECT_PAPER_DOC,
+ CLIENT_READ_PAPER_DOC,
+ CLIENT_RELATED_PAPERS_DOC,
+ CLIENT_SEARCH_GITHUB_DOC,
+ CLIENT_SEARCH_PAPERS_DOC,
+ doc,
+)
from .watcher import Watcher
# Kwargs that map to ScrapeOptions fields. Used by async crawl normalization
@@ -225,18 +233,24 @@ class FirecrawlClient:
) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection, profile, audit_metadata, integration]) else None
return scrape_module.scrape(self.http_client, url, options)
+ # Research paper index (/v2/search/research)
+ @doc(CLIENT_SEARCH_PAPERS_DOC)
def search_papers(self, query: str, **kwargs):
return research_module.search_papers(self.http_client, query, **kwargs)
+ @doc(CLIENT_INSPECT_PAPER_DOC)
def inspect_paper(self, paper_id: str):
return research_module.inspect_paper(self.http_client, paper_id)
+ @doc(CLIENT_READ_PAPER_DOC)
def read_paper(self, paper_id: str, query: str, **kwargs):
return research_module.read_paper(self.http_client, paper_id, query, **kwargs)
+ @doc(CLIENT_RELATED_PAPERS_DOC)
def related_papers(self, paper_id: str, intent: str, **kwargs):
return research_module.related_papers(self.http_client, paper_id, intent, **kwargs)
+ @doc(CLIENT_SEARCH_GITHUB_DOC)
def search_github(self, query: str, **kwargs):
return research_module.search_github(self.http_client, query, **kwargs)
diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.py
index 4b45e6207..a9db3e2c8 100644
--- a/apps/python-sdk/firecrawl/v2/client_async.py
+++ b/apps/python-sdk/firecrawl/v2/client_async.py
@@ -63,6 +63,14 @@ from .methods.aio import agent as async_agent # type: ignore[attr-defined]
from .methods.aio import browser as async_browser # type: ignore[attr-defined]
from .methods.aio import monitor as async_monitor # type: ignore[attr-defined]
from .methods.aio import research as async_research # type: ignore[attr-defined]
+from .methods.research_docs import (
+ ASYNC_CLIENT_INSPECT_PAPER_DOC,
+ ASYNC_CLIENT_READ_PAPER_DOC,
+ ASYNC_CLIENT_RELATED_PAPERS_DOC,
+ ASYNC_CLIENT_SEARCH_GITHUB_DOC,
+ ASYNC_CLIENT_SEARCH_PAPERS_DOC,
+ doc,
+)
from .client import _SCRAPE_OPTION_KEYS
from .watcher_async import AsyncWatcher
@@ -108,18 +116,24 @@ class AsyncFirecrawlClient:
options = ScrapeOptions(**{k: v for k, v in kwargs.items() if v is not None}) if kwargs else None
return await async_scrape.scrape(self.async_http_client, url, options)
+ # Research paper index (/v2/search/research)
+ @doc(ASYNC_CLIENT_SEARCH_PAPERS_DOC)
async def search_papers(self, query: str, **kwargs):
return await async_research.search_papers(self.async_http_client, query, **kwargs)
+ @doc(ASYNC_CLIENT_INSPECT_PAPER_DOC)
async def inspect_paper(self, paper_id: str):
return await async_research.inspect_paper(self.async_http_client, paper_id)
+ @doc(ASYNC_CLIENT_READ_PAPER_DOC)
async def read_paper(self, paper_id: str, query: str, **kwargs):
return await async_research.read_paper(self.async_http_client, paper_id, query, **kwargs)
+ @doc(ASYNC_CLIENT_RELATED_PAPERS_DOC)
async def related_papers(self, paper_id: str, intent: str, **kwargs):
return await async_research.related_papers(self.async_http_client, paper_id, intent, **kwargs)
+ @doc(ASYNC_CLIENT_SEARCH_GITHUB_DOC)
async def search_github(self, query: str, **kwargs):
return await async_research.search_github(self.async_http_client, query, **kwargs)
diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/research.py b/apps/python-sdk/firecrawl/v2/methods/aio/research.py
index 07897c9ba..e8725987a 100644
--- a/apps/python-sdk/firecrawl/v2/methods/aio/research.py
+++ b/apps/python-sdk/firecrawl/v2/methods/aio/research.py
@@ -1,5 +1,30 @@
"""
Async research functionality for Firecrawl v2 API.
+
+These functions query Firecrawl's **research paper index** (~43M paper
+abstracts) served at ``/v2/search/research``. The corpus is roughly 90%
+biomedical and life sciences β€” PubMed, bioRxiv and medRxiv β€” with arXiv
+covering physics, mathematics and computer science.
+
+.. warning::
+ This is **not** the same thing as ``search(categories=["research"])``.
+ That option is a website/domain filter applied to ordinary web search: it
+ restricts Google-style results to about 14 academic domains
+ (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, sciencedirect.com, ...)
+ and returns web page snippets. The functions in this module query the
+ paper index itself and return ranked paper records with full abstracts,
+ passage-level reads and citation-graph neighbours.
+
+ Use ``search_papers()`` for literature search; use
+ ``search(categories=["research"])`` when you want ordinary web results
+ narrowed to academic sites.
+
+.. note::
+ **Response keys are camelCase.** Unlike the rest of the Python SDK, these
+ functions return the raw JSON body from the API as a ``dict``: it is not
+ parsed into typed models and it is **not** normalized to snake_case. Expect
+ ``paperId``, ``primaryId``, ``createdDate``, ``updateDate``,
+ ``articleRank``, ``seedOverlap``, ``poolSize`` and so on.
"""
from typing import Any, Dict, List, Optional
@@ -8,6 +33,14 @@ from urllib.parse import quote
from ...utils import handle_response_error
from ...utils.http_client_async import AsyncHttpClient
from ...utils.get_version import get_version
+from ..research_docs import (
+ AIO_INSPECT_PAPER_DOC,
+ AIO_READ_PAPER_DOC,
+ AIO_RELATED_PAPERS_DOC,
+ AIO_SEARCH_GITHUB_DOC,
+ AIO_SEARCH_PAPERS_DOC,
+ doc,
+)
BASE = "/v2/search/research"
@@ -33,6 +66,7 @@ async def _get(client: AsyncHttpClient, path: str) -> Dict[str, Any]:
return response.json()
+@doc(AIO_SEARCH_PAPERS_DOC)
async def search_papers(
client: AsyncHttpClient,
query: str,
@@ -61,10 +95,15 @@ async def search_papers(
)
+@doc(AIO_INSPECT_PAPER_DOC)
async def inspect_paper(client: AsyncHttpClient, paper_id: str) -> Dict[str, Any]:
- return await _get(client, f"{BASE}/papers/{quote(paper_id, safe='')}")
+ return await _get(
+ client,
+ f"{BASE}/papers/{quote(paper_id, safe='')}" + _query({"origin": ORIGIN}),
+ )
+@doc(AIO_READ_PAPER_DOC)
async def read_paper(
client: AsyncHttpClient,
paper_id: str,
@@ -79,6 +118,7 @@ async def read_paper(
)
+@doc(AIO_RELATED_PAPERS_DOC)
async def related_papers(
client: AsyncHttpClient,
paper_id: str,
@@ -105,6 +145,7 @@ async def related_papers(
)
+@doc(AIO_SEARCH_GITHUB_DOC)
async def search_github(
client: AsyncHttpClient,
query: str,
diff --git a/apps/python-sdk/firecrawl/v2/methods/research.py b/apps/python-sdk/firecrawl/v2/methods/research.py
index 3255396ac..79d4d92cc 100644
--- a/apps/python-sdk/firecrawl/v2/methods/research.py
+++ b/apps/python-sdk/firecrawl/v2/methods/research.py
@@ -1,5 +1,30 @@
"""
Research functionality for Firecrawl v2 API.
+
+These functions query Firecrawl's **research paper index** (~43M paper
+abstracts) served at ``/v2/search/research``. The corpus is roughly 90%
+biomedical and life sciences β€” PubMed, bioRxiv and medRxiv β€” with arXiv
+covering physics, mathematics and computer science.
+
+.. warning::
+ This is **not** the same thing as ``search(categories=["research"])``.
+ That option is a website/domain filter applied to ordinary web search: it
+ restricts Google-style results to about 14 academic domains
+ (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, sciencedirect.com, ...)
+ and returns web page snippets. The functions in this module query the
+ paper index itself and return ranked paper records with full abstracts,
+ passage-level reads and citation-graph neighbours.
+
+ Use ``search_papers()`` for literature search; use
+ ``search(categories=["research"])`` when you want ordinary web results
+ narrowed to academic sites.
+
+.. note::
+ **Response keys are camelCase.** Unlike the rest of the Python SDK, these
+ functions return the raw JSON body from the API as a ``dict``: it is not
+ parsed into typed models and it is **not** normalized to snake_case. Expect
+ ``paperId``, ``primaryId``, ``createdDate``, ``updateDate``,
+ ``articleRank``, ``seedOverlap``, ``poolSize`` and so on.
"""
from typing import Any, Dict, List, Optional
@@ -7,6 +32,14 @@ from urllib.parse import quote
from ..utils import HttpClient, handle_response_error
from ..utils.get_version import get_version
+from .research_docs import (
+ INSPECT_PAPER_DOC,
+ READ_PAPER_DOC,
+ RELATED_PAPERS_DOC,
+ SEARCH_GITHUB_DOC,
+ SEARCH_PAPERS_DOC,
+ doc,
+)
BASE = "/v2/search/research"
@@ -32,6 +65,7 @@ def _get(client: HttpClient, path: str) -> Dict[str, Any]:
return response.json()
+@doc(SEARCH_PAPERS_DOC)
def search_papers(
client: HttpClient,
query: str,
@@ -60,10 +94,15 @@ def search_papers(
)
+@doc(INSPECT_PAPER_DOC)
def inspect_paper(client: HttpClient, paper_id: str) -> Dict[str, Any]:
- return _get(client, f"{BASE}/papers/{quote(paper_id, safe='')}")
+ return _get(
+ client,
+ f"{BASE}/papers/{quote(paper_id, safe='')}" + _query({"origin": ORIGIN}),
+ )
+@doc(READ_PAPER_DOC)
def read_paper(
client: HttpClient,
paper_id: str,
@@ -78,6 +117,7 @@ def read_paper(
)
+@doc(RELATED_PAPERS_DOC)
def related_papers(
client: HttpClient,
paper_id: str,
@@ -104,6 +144,7 @@ def related_papers(
)
+@doc(SEARCH_GITHUB_DOC)
def search_github(
client: HttpClient,
query: str,
diff --git a/apps/python-sdk/firecrawl/v2/methods/research_docs.py b/apps/python-sdk/firecrawl/v2/methods/research_docs.py
new file mode 100644
index 000000000..57a4d9c43
--- /dev/null
+++ b/apps/python-sdk/firecrawl/v2/methods/research_docs.py
@@ -0,0 +1,315 @@
+"""
+Canonical docstrings for the research paper index (``/v2/search/research``).
+
+Single source of truth for the prose that documents ``search_papers``,
+``inspect_paper``, ``read_paper``, ``related_papers`` and ``search_github``.
+The same five methods are surfaced at six layers β€” the implementation functions
+in ``research.py`` and ``aio/research.py``, and the delegating methods on
+``FirecrawlClient``, ``AsyncFirecrawlClient``, ``Firecrawl`` and
+``AsyncFirecrawl``. Every layer keeps a full docstring (``help()`` on any of
+them shows the complete text), but the text lives here once, so a correction to
+a factual claim β€” corpus size and composition, the
+``search(categories=["research"])`` disambiguation, wire names, Args/Returns β€”
+is a one-place edit instead of a six-way edit that drifts.
+
+Apply a constant with the :func:`doc` decorator::
+
+ @doc(SEARCH_PAPERS_DOC)
+ def search_papers(client, query, ...):
+ ...
+
+``*_DOC`` documents the module-level functions, which take an explicit
+``client``; ``CLIENT_*_DOC`` documents the client methods, which forward
+``**kwargs``. The ``AIO_*`` and ``ASYNC_CLIENT_*`` variants are the same text
+with the async wording substituted, never a separate copy of the prose.
+"""
+
+from typing import Callable, TypeVar
+
+F = TypeVar("F", bound=Callable)
+
+
+def doc(text: str) -> Callable[[F], F]:
+ """Attach ``text`` as the docstring of the decorated function.
+
+ The function itself is returned unchanged β€” this only sets ``__doc__``.
+ """
+
+ def decorate(fn: F) -> F:
+ fn.__doc__ = text
+ return fn
+
+ return decorate
+
+
+# Placeholders that the sync/async variants below substitute. They are not
+# prose, so a template can never accidentally ship with one unresolved.
+_CLIENT_ARG = "|CLIENT_ARG|"
+_AWAIT = "|AWAIT|"
+
+_SYNC_CLIENT_ARG = "client: HTTP client."
+_ASYNC_CLIENT_ARG = "client: Async HTTP client."
+
+
+# ---------------------------------------------------------------------------
+# Implementation functions (``research.py`` / ``aio/research.py``)
+# ---------------------------------------------------------------------------
+
+_SEARCH_PAPERS_TEMPLATE = """
+Search the research paper index by abstract relevance.
+
+Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of the
+corpus β€” biomedical and life sciences) plus arXiv (physics, mathematics,
+computer science). Semantic search over abstracts, not keyword matching.
+
+This is **not** ``search(categories=["research"])``. That option only
+restricts ordinary web search to ~14 academic websites and returns page
+snippets; this function searches the paper index and returns paper records.
+
+Args:
+ |CLIENT_ARG|
+ query: Natural-language query, e.g. ``"CRISPR base editing off-target
+ effects in primary human T cells"``.
+ k: Maximum number of papers to return.
+ authors: Filter by author name(s). Repeated per value.
+ categories: Filter by arXiv-style subject categories (e.g. ``["q-bio.GN"]``).
+ Note this is the *paper* category filter, unrelated to the
+ ``categories`` argument of ``search()``.
+ from_date: Inclusive lower bound on publication date (``YYYY-MM-DD``).
+ to_date: Inclusive upper bound on publication date (``YYYY-MM-DD``).
+
+Returns:
+ Raw API ``dict`` with ``success`` and ``results``. Keys are camelCase
+ and are **not** normalized to snake_case: each result carries
+ ``paperId``, ``primaryId`` (e.g. ``pmid:<id>``, ``doi:<id>``,
+ ``arxiv:<id>``), ``ids``, ``title``, ``abstract`` and ``score``.
+
+Example:
+ >>> res = search_papers(client, "tau aggregation inhibitors in Alzheimer's", k=10)
+ >>> res["results"][0]["paperId"]
+
+See Also:
+ ``inspect_paper``, ``read_paper``, ``related_papers``.
+"""
+
+_INSPECT_PAPER_TEMPLATE = """
+Fetch metadata for a single paper in the research paper index.
+
+Resolves against the same ~43M-abstract corpus as ``search_papers``
+(PubMed / bioRxiv / medRxiv / arXiv).
+
+Args:
+ |CLIENT_ARG|
+ paper_id: A canonical ``paperId`` returned by ``search_papers``, or a
+ namespaced id key such as ``pmid:<id>``, ``pmcid:<id>``,
+ ``doi:<doi>`` or ``arxiv:<id>``. Bare arXiv ids and arXiv URLs are
+ also accepted.
+
+Returns:
+ Raw API ``dict`` with ``success`` and ``paper``. Keys are camelCase and
+ are **not** normalized to snake_case β€” expect ``paperId``, ``ids``,
+ ``title``, ``abstract``, ``authors``, ``categories``, ``createdDate``,
+ ``updateDate``.
+
+See Also:
+ ``read_paper`` to search inside the body of a paper.
+"""
+
+_READ_PAPER_TEMPLATE = """
+Read inside a paper: return the passages of its body that best match a query.
+
+Full-text passage retrieval over the research paper index (PubMed /
+bioRxiv / medRxiv / arXiv). Use this to answer a specific question against
+one paper instead of re-reading the whole document.
+
+Args:
+ |CLIENT_ARG|
+ paper_id: Canonical ``paperId`` or a namespaced id key
+ (``pmid:<id>``, ``pmcid:<id>``, ``doi:<doi>``, ``arxiv:<id>``).
+ query: What to look for inside the paper, e.g. ``"primary endpoint and
+ hazard ratio"``.
+ k: Maximum number of passages to return.
+
+Returns:
+ Raw API ``dict`` with ``success``, ``paper``, ``paperId``, ``query`` and
+ ``passages`` (each ``{"text": ..., "score": ...}``). Keys are camelCase
+ and are **not** normalized to snake_case.
+"""
+
+_RELATED_PAPERS_TEMPLATE = """
+Find papers related to a seed paper via the citation graph.
+
+Walks citations/references within the research paper index (PubMed /
+bioRxiv / medRxiv / arXiv) and re-ranks candidates against your stated
+``intent``, so "related" means related *for your purpose*, not merely
+co-cited.
+
+Args:
+ |CLIENT_ARG|
+ paper_id: Seed paper β€” canonical ``paperId`` or a namespaced id key
+ (``pmid:<id>``, ``pmcid:<id>``, ``doi:<doi>``, ``arxiv:<id>``).
+ intent: Required. What you want the related papers *for*, e.g.
+ ``"replication attempts in larger cohorts"``. Used to re-rank.
+ mode: Traversal mode over the citation graph, e.g. ``"citers"``.
+ k: Maximum number of papers to return.
+ rerank: Whether to apply the intent reranker. Serialized as
+ ``"true"``/``"false"``.
+ anchor: Additional seed papers to anchor the neighbourhood on.
+
+Returns:
+ Raw API ``dict`` with ``success``, ``results``, ``poolSize``,
+ ``truncated`` and optional ``note``. Keys are camelCase and are **not**
+ normalized to snake_case; each result carries a ``signals`` object with
+ ``structural``, ``semantic``, ``articleRank`` and ``seedOverlap``.
+
+Note:
+ This is the Python name for the endpoint the JS SDK exposes as
+ ``research.similarPapers()``. Same endpoint
+ (``/v2/search/research/papers/{id}/similar``), different method name.
+"""
+
+_SEARCH_GITHUB_TEMPLATE = """
+Search the developer index: GitHub issue/PR history and repository readmes.
+
+This is the code-and-discussion companion to ``search_papers`` and is served
+by the same ``/v2/search/research`` surface. It searches indexed GitHub
+history and readmes β€” it does **not** search the paper corpus, and it is not
+the same as ``search(categories=["github"])`` (which is a ``site:github.com``
+filter on ordinary web search).
+
+Args:
+ |CLIENT_ARG|
+ query: Natural-language query, e.g. ``"pysam VCF parsing memory leak"``.
+ k: Maximum number of results to return.
+
+Returns:
+ Raw API ``dict`` with ``success`` and ``results``. Keys are camelCase and
+ are **not** normalized to snake_case β€” expect ``resultType``, ``repo``,
+ ``url``, ``pageType``, ``number`` and a ``scoreBreakdown`` object.
+"""
+
+SEARCH_PAPERS_DOC = _SEARCH_PAPERS_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG)
+AIO_SEARCH_PAPERS_DOC = _SEARCH_PAPERS_TEMPLATE.replace(_CLIENT_ARG, _ASYNC_CLIENT_ARG)
+INSPECT_PAPER_DOC = _INSPECT_PAPER_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG)
+AIO_INSPECT_PAPER_DOC = _INSPECT_PAPER_TEMPLATE.replace(_CLIENT_ARG, _ASYNC_CLIENT_ARG)
+READ_PAPER_DOC = _READ_PAPER_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG)
+AIO_READ_PAPER_DOC = _READ_PAPER_TEMPLATE.replace(_CLIENT_ARG, _ASYNC_CLIENT_ARG)
+RELATED_PAPERS_DOC = _RELATED_PAPERS_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG)
+AIO_RELATED_PAPERS_DOC = _RELATED_PAPERS_TEMPLATE.replace(_CLIENT_ARG, _ASYNC_CLIENT_ARG)
+SEARCH_GITHUB_DOC = _SEARCH_GITHUB_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG)
+AIO_SEARCH_GITHUB_DOC = _SEARCH_GITHUB_TEMPLATE.replace(_CLIENT_ARG, _ASYNC_CLIENT_ARG)
+
+
+# ---------------------------------------------------------------------------
+# Client methods (``FirecrawlClient``, ``AsyncFirecrawlClient``, ``Firecrawl``,
+# ``AsyncFirecrawl``) β€” same prose, wrapper-shaped Args.
+# ---------------------------------------------------------------------------
+
+_CLIENT_SEARCH_PAPERS_TEMPLATE = """
+Search the research paper index by abstract relevance.
+
+Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of
+the corpus β€” biomedical and life sciences) plus arXiv (physics,
+mathematics, computer science). Use this for literature search.
+
+Not to be confused with ``search(categories=["research"])``: that is a
+website/domain filter on ordinary web search (it narrows Google-style
+results to ~14 academic domains and returns page snippets). This method
+searches the paper index itself and returns ranked paper records.
+
+Args:
+ query: Natural-language query, e.g. ``"CRISPR base editing
+ off-target effects in primary human T cells"``.
+ **kwargs: ``k``, ``authors``, ``categories``, ``from_date``,
+ ``to_date``.
+
+Returns:
+ Raw API ``dict`` with ``success`` and ``results``. These research
+ responses are **not** normalized to snake_case like the rest of the
+ SDK β€” keys are camelCase (``paperId``, ``primaryId``, ...).
+
+Example:
+ >>> |AWAIT|firecrawl.search_papers("GLP-1 receptor agonists cardiovascular outcomes", k=10)
+"""
+
+_CLIENT_INSPECT_PAPER_TEMPLATE = """
+Fetch metadata for one paper in the research paper index.
+
+Args:
+ paper_id: Canonical ``paperId`` from ``search_papers``, or a
+ namespaced id key such as ``pmid:<id>``, ``pmcid:<id>``,
+ ``doi:<doi>`` or ``arxiv:<id>``.
+
+Returns:
+ Raw API ``dict`` with ``success`` and ``paper``. Keys are camelCase,
+ **not** snake_case-normalized (``paperId``, ``createdDate``, ...).
+"""
+
+_CLIENT_READ_PAPER_TEMPLATE = """
+Read inside a paper: return the body passages that best match a query.
+
+Full-text passage retrieval over the research paper index (PubMed /
+bioRxiv / medRxiv / arXiv).
+
+Args:
+ paper_id: Canonical ``paperId`` or namespaced id key
+ (``pmid:<id>``, ``pmcid:<id>``, ``doi:<doi>``, ``arxiv:<id>``).
+ query: What to look for inside the paper, e.g. ``"primary endpoint
+ and hazard ratio"``.
+ **kwargs: ``k`` (max passages).
+
+Returns:
+ Raw API ``dict`` with ``success``, ``paper``, ``paperId``, ``query``
+ and ``passages``. Keys are camelCase, **not** snake_case-normalized.
+"""
+
+_CLIENT_RELATED_PAPERS_TEMPLATE = """
+Find papers related to a seed paper via the citation graph.
+
+Candidates are re-ranked against ``intent``, so "related" means related
+for your stated purpose rather than merely co-cited.
+
+Args:
+ paper_id: Seed paper β€” canonical ``paperId`` or namespaced id key
+ (``pmid:<id>``, ``pmcid:<id>``, ``doi:<doi>``, ``arxiv:<id>``).
+ intent: What you want the related papers for, e.g. ``"replication
+ attempts in larger cohorts"``.
+ **kwargs: ``mode``, ``k``, ``rerank``, ``anchor``.
+
+Returns:
+ Raw API ``dict`` with ``success``, ``results``, ``poolSize``,
+ ``truncated`` and optional ``note``. Keys are camelCase, **not**
+ snake_case-normalized (``articleRank``, ``seedOverlap``, ...).
+
+Note:
+ The JS SDK exposes this same endpoint as
+ ``research.similarPapers()``.
+"""
+
+_CLIENT_SEARCH_GITHUB_TEMPLATE = """
+Search the developer index: GitHub issue/PR history and repo readmes.
+
+The code-and-discussion companion to ``search_papers``, served by the
+same ``/v2/search/research`` surface. It does not search the paper
+corpus, and it is not ``search(categories=["github"])`` (which is just a
+``site:github.com`` filter on ordinary web search).
+
+Args:
+ query: Natural-language query, e.g. ``"pysam VCF parsing memory leak"``.
+ **kwargs: ``k``.
+
+Returns:
+ Raw API ``dict`` with ``success`` and ``results``. Keys are
+ camelCase, **not** snake_case-normalized.
+"""
+
+CLIENT_SEARCH_PAPERS_DOC = _CLIENT_SEARCH_PAPERS_TEMPLATE.replace(_AWAIT, "")
+ASYNC_CLIENT_SEARCH_PAPERS_DOC = _CLIENT_SEARCH_PAPERS_TEMPLATE.replace(_AWAIT, "await ")
+CLIENT_INSPECT_PAPER_DOC = _CLIENT_INSPECT_PAPER_TEMPLATE.replace(_AWAIT, "")
+ASYNC_CLIENT_INSPECT_PAPER_DOC = _CLIENT_INSPECT_PAPER_TEMPLATE.replace(_AWAIT, "await ")
+CLIENT_READ_PAPER_DOC = _CLIENT_READ_PAPER_TEMPLATE.replace(_AWAIT, "")
+ASYNC_CLIENT_READ_PAPER_DOC = _CLIENT_READ_PAPER_TEMPLATE.replace(_AWAIT, "await ")
+CLIENT_RELATED_PAPERS_DOC = _CLIENT_RELATED_PAPERS_TEMPLATE.replace(_AWAIT, "")
+ASYNC_CLIENT_RELATED_PAPERS_DOC = _CLIENT_RELATED_PAPERS_TEMPLATE.replace(_AWAIT, "await ")
+CLIENT_SEARCH_GITHUB_DOC = _CLIENT_SEARCH_GITHUB_TEMPLATE.replace(_AWAIT, "")
+ASYNC_CLIENT_SEARCH_GITHUB_DOC = _CLIENT_SEARCH_GITHUB_TEMPLATE.replace(_AWAIT, "await ")
diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.py
index 530044981..8039d3395 100644
--- a/apps/python-sdk/firecrawl/v2/types.py
+++ b/apps/python-sdk/firecrawl/v2/types.py
@@ -559,12 +559,30 @@ SourceOption = Union[str, Source]
class Category(BaseModel):
"""Configuration for a search category.
+ Categories narrow ordinary **web search**. They do not switch `search()` to
+ a different index.
+
Supported categories:
- - "github": Filter results to GitHub repositories
- - "research": Filter results to research papers and academic sites
+ - "github": Restrict web results to github.com (a `site:` filter)
+ - "research": Restrict web results to a fixed list of ~14 academic
+ *websites* (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, science.org,
+ ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). This is a
+ website/domain filter and it returns ordinary web page results for those
+ domains β€” **not** paper records.
- "pdf": Filter results to PDF files (adds filetype:pdf to search)
- "developer": Add developer results (issues, pull requests, READMEs and
documentation) under `.developer`
+
+ .. warning::
+ ``categories=["research"]`` is **not** Firecrawl's research paper index.
+ To search papers themselves β€” ~43M abstracts, about 90% biomedical
+ (PubMed, bioRxiv, medRxiv) plus arXiv β€” with full abstracts, in-body
+ passage reads and citation-graph expansion, use
+ :meth:`Firecrawl.search_papers` (and ``inspect_paper``, ``read_paper``,
+ ``related_papers``), which call ``/v2/search/research``.
+
+ Rule of thumb: literature search β†’ ``search_papers()``; web pages that
+ happen to live on academic domains β†’ ``search(categories=["research"])``.
"""
type: str
diff --git a/apps/python-sdk/pyproject.toml b/apps/python-sdk/pyproject.toml
index 8c9925118..9e3017afe 100644
--- a/apps/python-sdk/pyproject.toml
+++ b/apps/python-sdk/pyproject.toml
@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"
[project]
dynamic = ["version"]
name = "firecrawl-py"
-description = "Python SDK for Firecrawl API"
+description = "Python SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts"
readme = {file="README.md", content-type = "text/markdown"}
requires-python = ">=3.8"
dependencies = [
@@ -41,9 +41,33 @@ classifiers = [
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: Text Processing",
"Topic :: Text Processing :: Indexing",
+ "Intended Audience :: Science/Research",
+ "Topic :: Scientific/Engineering",
+ "Topic :: Scientific/Engineering :: Bio-Informatics",
+ "Topic :: Scientific/Engineering :: Medical Science Apps.",
]
-keywords = ["SDK", "API", "firecrawl"]
+keywords = [
+ "SDK",
+ "API",
+ "firecrawl",
+ "web scraping",
+ "crawler",
+ "web search",
+ "literature search",
+ "research",
+ "scientific papers",
+ "academic search",
+ "biomedical",
+ "life sciences",
+ "pubmed",
+ "biorxiv",
+ "medrxiv",
+ "arxiv",
+ "preprints",
+ "citations",
+ "bioinformatics",
+]
[project.urls]
"Documentation" = "https://docs.firecrawl.dev"
diff --git a/apps/python-sdk/setup.py b/apps/python-sdk/setup.py
index 79f3922b4..c705e8c51 100644
--- a/apps/python-sdk/setup.py
+++ b/apps/python-sdk/setup.py
@@ -22,7 +22,7 @@ setup(
url="https://github.com/firecrawl/firecrawl",
author="Mendable.ai",
author_email="[email protected]",
- description="Python SDK for Firecrawl API",
+ description="Python SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",
long_description=long_description_content,
long_description_content_type="text/markdown",
packages=find_packages(),
@@ -57,8 +57,16 @@ setup(
"Topic :: Software Development :: Libraries :: Python Modules",
"Topic :: Text Processing",
"Topic :: Text Processing :: Indexing",
+ "Intended Audience :: Science/Research",
+ "Topic :: Scientific/Engineering",
+ "Topic :: Scientific/Engineering :: Bio-Informatics",
+ "Topic :: Scientific/Engineering :: Medical Science Apps.",
],
- keywords="SDK API firecrawl",
+ keywords=(
+ "SDK API firecrawl web scraping crawler web search literature search "
+ "research scientific papers academic search biomedical life sciences "
+ "pubmed biorxiv medrxiv arxiv preprints citations bioinformatics"
+ ),
project_urls={
"Documentation": "https://docs.firecrawl.dev",
"Source": "https://github.com/firecrawl/firecrawl",