Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(sdks): research API accuracy, unified-client methods, origin telemetry (#4289)
Himadri Mishra8373dab922de
diff --git a/apps/js-sdk/firecrawl/README.md b/apps/js-sdk/firecrawl/README.mdindex 982bf4348..c0d9aa5c8 100644--- a/apps/js-sdk/firecrawl/README.md+++ b/apps/js-sdk/firecrawl/README.md@@ -166,6 +166,73 @@ const mapResult = await app.map('https://example.com');console.log(mapResult);```+### Search++Use `search` to search the web and optionally scrape the results in the same call.++```js+const results = await app.search('what is retrieval augmented generation?', { limit: 5 });+for (const result of results.web ?? []) {+ console.log(result.url, '-', result.title);+}++// Scrape every result as part of the search:+const scraped = await app.search('firecrawl changelog', {+ limit: 3,+ scrapeOptions: { formats: ['markdown'] },+});+```++Results are grouped by source: `.web`, `.news`, `.images` and `.developer`.++Use `categories` to narrow web search to a kind of site:++```js+const results = await app.search('nanopore basecalling accuracy', {+ categories: ['research'],+});+```++> **`categories: ['research']` is a website filter, not the paper index.** It+> restricts ordinary web search to roughly 14 academic domains (arxiv.org,+> pubmed.ncbi.nlm.nih.gov, nature.com, biorxiv.org, ...) and returns web page+> results. To search papers themselves, use `research.searchPapers` below.++### Research / paper search++Use `app.research` to search Firecrawl's research paper index: ~43M paper+abstracts, roughly 90% biomedical and life sciences (PubMed, bioRxiv, medRxiv),+plus arXiv for physics, mathematics and computer science.++```js+// Search the paper index (semantic search over abstracts):+const papers = await app.research.searchPapers(+ 'CRISPR base editing off-target effects in primary human T cells',+ { k: 10 },+);+for (const paper of papers.results) {+ console.log(paper.primaryId, '-', paper.title);+}++// Inspect one paper's metadata (accepts pmid:, pmcid:, doi: or arxiv: ids):+const paper = await app.research.getPaper('pmid:<id>');++// Read the passages inside a paper that answer a specific question:+const read = await app.research.getPaper('pmid:<id>', {+ query: 'what was the primary endpoint and the reported hazard ratio?',+ k: 4,+});++// Expand along the citation graph, re-ranked for your stated intent:+const related = await app.research.similarPapers('pmid:<id>', {+ intent: 'replication attempts in larger cohorts',+ k: 20,+});+```++A companion `app.research.searchGithub` searches indexed GitHub issue/PR history+and repository readmes.+### Scrape-bound interactive browsing (v2)Use a scrape job ID to keep interacting with the replayed browser context:diff --git a/apps/js-sdk/firecrawl/package.json b/apps/js-sdk/firecrawl/package.jsonindex b67b98ab5..ac1296803 100644--- a/apps/js-sdk/firecrawl/package.json+++ b/apps/js-sdk/firecrawl/package.json@@ -1,7 +1,7 @@{"name": "@mendable/firecrawl-js",- "version": "4.32.0",- "description": "JavaScript SDK for Firecrawl API",+ "version": "4.32.1",+ "description": "JavaScript SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts","main": "dist/index.js","types": "dist/index.d.ts","exports": {@@ -57,7 +57,21 @@"web","scraper","api",- "sdk"+ "sdk",+ "web-search",+ "literature-search",+ "research",+ "scientific-papers",+ "academic-search",+ "biomedical",+ "life-sciences",+ "pubmed",+ "biorxiv",+ "medrxiv",+ "arxiv",+ "preprints",+ "citations",+ "bioinformatics"],"engines": {"node": ">=22.0.0"diff --git a/apps/js-sdk/firecrawl/src/v1/index.ts b/apps/js-sdk/firecrawl/src/v1/index.tsindex 926919657..e318eaf38 100644--- a/apps/js-sdk/firecrawl/src/v1/index.ts+++ b/apps/js-sdk/firecrawl/src/v1/index.ts@@ -1642,7 +1642,7 @@ export default class FirecrawlApp {* @param onActivity - Optional callback to receive activity updates in real-time.* @param onSource - Optional callback to receive source updates in real-time.* @returns The final research results.- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.*/async deepResearch(query: string,@@ -1730,7 +1730,7 @@ export default class FirecrawlApp {* Initiates a deep research operation on a given query without polling.* @param params - Parameters for the deep research operation.* @returns The response containing the research job ID.- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.*/async asyncDeepResearch(query: string, params: DeepResearchParams<zt.ZodSchema>): Promise<DeepResearchResponse | ErrorResponse> {const headers = this.prepareHeaders();@@ -1772,7 +1772,7 @@ export default class FirecrawlApp {* Checks the status of a deep research operation.* @param id - The ID of the deep research operation.* @returns The current status and results of the research operation.- * @deprecated /v1/deep-research is deprecated. Use /v2/search instead.+ * @deprecated /v1/deep-research is deprecated. Use /v2/search for web research, or research.searchPapers() for scientific literature.*/async checkDeepResearchStatus(id: string): Promise<DeepResearchStatusResponse | ErrorResponse> {const headers = this.prepareHeaders();diff --git a/apps/js-sdk/firecrawl/src/v2/client.ts b/apps/js-sdk/firecrawl/src/v2/client.tsindex 42273692b..f7035cc2b 100644--- a/apps/js-sdk/firecrawl/src/v2/client.ts+++ b/apps/js-sdk/firecrawl/src/v2/client.ts@@ -261,8 +261,19 @@ export class FirecrawlClient {// Research/**- * Access the v2 research endpoints (arXiv papers + GitHub history/readmes).- * Example: `firecrawl.research.searchPapers("diffusion models")`.+ * Access the v2 research endpoints β Firecrawl's **research paper index**+ * (~43M paper abstracts) plus GitHub history/readmes.+ *+ * The paper corpus is roughly 90% biomedical and life sciences β PubMed,+ * bioRxiv and medRxiv β with arXiv covering physics, mathematics and+ * computer science.+ *+ * β οΈ Not the same as `search({ categories: ["research"] })`, which is only a+ * website/domain filter over ordinary web search (~14 academic domains,+ * returning page snippets). Use `research.searchPapers()` for literature+ * search.+ *+ * Example: `firecrawl.research.searchPapers("GLP-1 receptor agonists cardiovascular outcomes")`.*/get research(): ResearchClient {if (!this._research) this._research = new ResearchClient(this.http);diff --git a/apps/js-sdk/firecrawl/src/v2/methods/research.ts b/apps/js-sdk/firecrawl/src/v2/methods/research.tsindex ffb9494ee..a05d32249 100644--- a/apps/js-sdk/firecrawl/src/v2/methods/research.ts+++ b/apps/js-sdk/firecrawl/src/v2/methods/research.ts@@ -63,16 +63,47 @@ function normalizeResearchError(err: any, action: string): never {}/**- * Client for the v2 research endpoints (arXiv papers + GitHub history/readmes).+ * Client for the v2 research endpoints β Firecrawl's **research paper index**+ * (~43M paper abstracts) plus GitHub history/readmes.+ *+ * The paper corpus is roughly 90% biomedical and life sciences β PubMed,+ * bioRxiv and medRxiv β with arXiv covering physics, mathematics and computer+ * science. Supports abstract search, paper metadata, in-body passage reads and+ * citation-graph expansion.+ *+ * β οΈ This is **not** the same as `search({ categories: ["research"] })`. That+ * option is a website/domain filter on ordinary web search: it narrows+ * Google-style results to ~14 academic domains and returns page snippets. The+ * methods here query the paper index itself and return ranked paper records.+ ** Accessed via `firecrawl.research`.+ *+ * @example+ * ```ts+ * const res = await firecrawl.research.searchPapers(+ * "tau aggregation inhibitors in Alzheimer's disease",+ * { k: 10 },+ * );+ * ```*/export class ResearchClient {constructor(private readonly http: HttpClient) {}/**- * Search papers by abstract relevance.- * @param query Natural-language search query.- * @param options Optional filters (k, authors, categories, from, to).+ * Search the research paper index by abstract relevance.+ *+ * Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of the+ * corpus β biomedical and life sciences) plus arXiv (physics, mathematics,+ * computer science). Semantic search over abstracts, not keyword matching.+ *+ * This is **not** `search({ categories: ["research"] })`, which only narrows+ * ordinary web search to ~14 academic websites.+ *+ * @param query Natural-language search query, e.g. `"CRISPR base editing+ * off-target effects in primary human T cells"`.+ * @param options Optional filters (k, authors, categories, from, to). Note+ * `categories` here filters *paper* subject categories (e.g. `"q-bio.GN"`)+ * and is unrelated to the `categories` option of `search()`.*/async searchPapers(query: string,@@ -102,8 +133,9 @@ export class ResearchClient {/*** Get paper metadata (detail mode), or read in-body passages (when `query` is* supplied). `k` is only valid together with `query`.- * @param id Paper reference: a canonical `paper_id`, an `arxiv:<id>` key, or a- * bare arXiv id / URL.+ * @param id Paper reference: a canonical `paperId`, or a namespaced id key+ * such as `pmid:<id>`, `pmcid:<id>`, `doi:<doi>` or `arxiv:<id>`. Bare+ * arXiv ids and arXiv URLs are also accepted.* @param options Optional `query` (switches to read mode) and `k`.*/async getPaper(diff --git a/apps/js-sdk/firecrawl/src/v2/types.ts b/apps/js-sdk/firecrawl/src/v2/types.tsindex e1ec6c958..1f99c6232 100644--- a/apps/js-sdk/firecrawl/src/v2/types.ts+++ b/apps/js-sdk/firecrawl/src/v2/types.ts@@ -694,6 +694,28 @@ export interface SearchData {developer?: Array<SearchResultWeb | Document>;}+/**+ * Narrows ordinary **web search**. A category does not switch `search()` to a+ * different index.+ *+ * - `github` β restrict web results to github.com (a `site:` filter).+ * - `research` β restrict web results to a fixed list of ~14 academic+ * *websites* (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, science.org,+ * ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). It returns+ * ordinary web page results from those domains, **not** paper records.+ * - `pdf` β restrict results to PDFs (adds `filetype:pdf`).+ * - `developer` β add developer results (issues, pull requests, READMEs and+ * documentation) under `.developer`.+ *+ * β οΈ `categories: ["research"]` is **not** Firecrawl's research paper index.+ * To search papers themselves β ~43M abstracts, roughly 90% biomedical+ * (PubMed, bioRxiv, medRxiv) plus arXiv β with full abstracts, in-body passage+ * reads and citation-graph expansion, use `firecrawl.research.searchPapers()`+ * (plus `getPaper()` and `similarPapers()`), which call `/v2/search/research`.+ *+ * Rule of thumb: literature search β `research.searchPapers()`; web pages that+ * happen to live on academic domains β `search({ categories: ["research"] })`.+ */export interface CategoryOption {type: "github" | "research" | "pdf" | "developer";}@@ -703,6 +725,14 @@ export interface SearchRequest {sources?: Array<"web" | "news" | "images" | { type: "web" | "news" | "images" }>;+ /**+ * Narrow web search by category. See {@link CategoryOption}.+ *+ * β οΈ `"research"` is a website/domain filter over ordinary web search (~14+ * academic domains), **not** the research paper index. For literature search+ * over ~43M paper abstracts (PubMed / bioRxiv / medRxiv / arXiv) use+ * `firecrawl.research.searchPapers()` instead.+ */categories?: Array<"github" | "research" | "pdf" | "developer" | CategoryOption>;@@ -1362,8 +1392,20 @@ export interface BrowserListResponse {// ---------- Research (v2) ----------/**- * Source identifiers grouped by namespace. Currently only `arxiv` is- * populated; each value is an array of ids in that namespace.+ * Source identifiers grouped by namespace. Each key is a namespace and each+ * value is an array of ids in that namespace.+ *+ * Which namespaces appear depends on the paper's source. Biomedical records+ * (PubMed, bioRxiv, medRxiv) typically carry `pmid`, `pmcid` and/or `doi`;+ * arXiv records carry `arxiv`. The set is open β treat this as a lookup by+ * namespace rather than a fixed schema, and prefer {@link PaperResult.primaryId}+ * when you just need one citable identifier.+ *+ * @example+ * ```ts+ * const doi = paper.ids?.doi?.[0];+ * const pmid = paper.ids?.pmid?.[0];+ * ```*/export type IdMap = Record<string, string[]>;@@ -1379,7 +1421,7 @@ export interface PaperSignals {seedOverlap: number;}-/** A ranked paper. `paperId` is canonical; arXiv lives in `ids`. */+/** A ranked paper. `paperId` is canonical; source ids (pmid, pmcid, doi, arxiv, ...) live in `ids`. */export interface PaperResult {/** Canonical paper id β the Milvus INT64 primary key as a decimal string. */paperId: string;diff --git a/apps/python-sdk/README.md b/apps/python-sdk/README.mdindex 5c2a2e461..ad571ee4d 100644--- a/apps/python-sdk/README.md+++ b/apps/python-sdk/README.md@@ -180,6 +180,80 @@ map_result = firecrawl.map('https://firecrawl.dev')print(map_result)```+### Search++Use `search` to search the web and optionally scrape the results in the same call.++```python+# Search the web (v2):+results = firecrawl.search("what is retrieval augmented generation?", limit=5)+for result in results.web or []:+ print(result.url, "-", result.title)++# Scrape every result as part of the search:+results = firecrawl.search(+ "firecrawl changelog",+ limit=3,+ scrape_options={"formats": ["markdown"]},+)+```++Results are grouped by source: `.web`, `.news`, `.images` and `.developer`.++Use `categories` to narrow web search to a kind of site:++```python+results = firecrawl.search("nanopore basecalling accuracy", categories=["research"])+```++> **`categories=["research"]` is a website filter, not the paper index.** It+> restricts ordinary web search to roughly 14 academic domains (arxiv.org,+> pubmed.ncbi.nlm.nih.gov, nature.com, biorxiv.org, ...) and returns web page+> results. To search papers themselves, use `search_papers` below.++### Research / paper search++Use `search_papers` to search Firecrawl's research paper index: ~43M paper+abstracts, roughly 90% biomedical and life sciences (PubMed, bioRxiv, medRxiv),+plus arXiv for physics, mathematics and computer science.++```python+# Search the paper index (semantic search over abstracts):+papers = firecrawl.search_papers(+ "CRISPR base editing off-target effects in primary human T cells",+ k=10,+)+for paper in papers["results"]:+ print(paper["primaryId"], "-", paper["title"])++# Inspect one paper's metadata (accepts pmid:, pmcid:, doi: or arxiv: ids):+paper = firecrawl.inspect_paper("pmid:<id>")++# Read the passages inside a paper that answer a specific question:+passages = firecrawl.read_paper(+ "pmid:<id>",+ "what was the primary endpoint and the reported hazard ratio?",+ k=4,+)++# Expand along the citation graph, re-ranked for your stated intent:+related = firecrawl.related_papers(+ "pmid:<id>",+ intent="replication attempts in larger cohorts",+ k=20,+)+```++A companion `search_github` searches indexed GitHub issue/PR history and repo+readmes.++> **Response keys are camelCase.** Unlike the rest of the SDK, the research+> methods return the raw JSON body as a `dict` β they are not parsed into typed+> models and not normalized to snake_case. Expect `paperId`, `primaryId`,+> `createdDate`, `articleRank`, `poolSize`, and so on.++Every method above is also available on `AsyncFirecrawl` with the same name.+### Scrape-bound interactive browsing (v2)Use a scrape job ID to keep interacting with the replayed browser context:diff --git a/apps/python-sdk/firecrawl/__init__.py b/apps/python-sdk/firecrawl/__init__.pyindex 02bad5b9d..908e06d0d 100644--- a/apps/python-sdk/firecrawl/__init__.py+++ b/apps/python-sdk/firecrawl/__init__.py@@ -18,7 +18,7 @@ from .v1 import (V1ChangeTrackingOptions,)-__version__ = "4.34.0"+__version__ = "4.35.0"# Define the logger for the Firecrawl projectlogger: logging.Logger = logging.getLogger("firecrawl")diff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_research.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_research.pynew file mode 100644index 000000000..2e23065e9--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/aio/test_aio_research.py@@ -0,0 +1,286 @@+"""+Unit tests for the async v2 research paper index methods.++Async mirror of ``firecrawl/__tests__/unit/v2/methods/test_research.py``: the+async HTTP client is mocked and the tests pin the method names, the request+paths and the query-string construction. No network access.+"""++from urllib.parse import parse_qs, urlparse++import pytest++from firecrawl.v2.methods.aio import research as aio_research+from firecrawl.v2.utils.error_handler import BadRequestError, FirecrawlError+++class FakeResponse:+ """Minimal stand-in for an httpx/requests-style response."""++ def __init__(self, status_code=200, payload=None):+ self.status_code = status_code+ self._payload = {} if payload is None else payload+ self.text = str(self._payload)++ def json(self):+ return self._payload+++class FakeAsyncHttpClient:+ """Records every path requested through ``await get``."""++ def __init__(self, response=None):+ self.calls = []+ self._response = response or FakeResponse()++ async def get(self, path):+ self.calls.append(path)+ return self._response++ @property+ def last_path(self):+ return self.calls[-1]+++def split(path):+ """Split a recorded path into (path, {param: [values]})."""+ parsed = urlparse(path)+ return parsed.path, parse_qs(parsed.query, keep_blank_values=True)+++async def test_search_papers_builds_query_string_with_exploded_arrays():+ client = FakeAsyncHttpClient(FakeResponse(200, {"success": True, "results": []}))+ await aio_research.search_papers(+ client,+ "diffusion models",+ k=10,+ authors=["Ho", "Abbeel"],+ categories=["cs.LG", "stat.ML"],+ from_date="2020-01-01",+ to_date="2024-12-31",+ )+ path, qs = split(client.last_path)+ assert path == "/v2/search/research/papers"+ assert qs["query"] == ["diffusion models"]+ assert qs["k"] == ["10"]+ assert qs["authors"] == ["Ho", "Abbeel"]+ assert qs["categories"] == ["cs.LG", "stat.ML"]+ assert qs["from"] == ["2020-01-01"]+ assert qs["to"] == ["2024-12-31"]+++async def test_search_papers_omits_absent_options():+ client = FakeAsyncHttpClient(FakeResponse(200, {"success": True, "results": []}))+ await aio_research.search_papers(client, "q")+ path, qs = split(client.last_path)+ assert path == "/v2/search/research/papers"+ assert sorted(qs.keys()) == ["origin", "query"]+ assert qs["origin"][0].startswith("python-sdk@")+++async def test_search_papers_returns_body_verbatim_in_camel_case():+ payload = {+ "success": True,+ "results": [{"paperId": "1", "primaryId": "pmid:12345678", "score": 0.5}],+ }+ client = FakeAsyncHttpClient(FakeResponse(200, payload))+ result = await aio_research.search_papers(client, "q")+ assert result == payload+ assert "paperId" in result["results"][0]+ assert "paper_id" not in result["results"][0]+++ "paper_id, expected_path",+ [+ ("arxiv:2105.05233", "/v2/search/research/papers/arxiv%3A2105.05233"),+ ("pmid:12345678", "/v2/search/research/papers/pmid%3A12345678"),+ ("pmcid:PMC1234567", "/v2/search/research/papers/pmcid%3APMC1234567"),+ (+ "doi:10.1038/s41586-020-2649-2",+ "/v2/search/research/papers/doi%3A10.1038%2Fs41586-020-2649-2",+ ),+ ],+)+async def test_inspect_paper_encodes_namespaced_ids(paper_id, expected_path):+ client = FakeAsyncHttpClient(FakeResponse(200, {"success": True, "paper": {}}))+ await aio_research.inspect_paper(client, paper_id)+ path, qs = split(client.last_path)+ assert path == expected_path+ # detail mode carries origin, like the other four research methods+ assert sorted(qs.keys()) == ["origin"]+ assert qs["origin"][0].startswith("python-sdk@")+++async def test_read_paper_adds_query_and_k():+ client = FakeAsyncHttpClient(+ FakeResponse(200, {"success": True, "paper": {}, "passages": []})+ )+ await aio_research.read_paper(client, "pmid:12345678", "primary endpoint", k=4)+ path, qs = split(client.last_path)+ assert path == "/v2/search/research/papers/pmid%3A12345678"+ assert qs["query"] == ["primary endpoint"]+ assert qs["k"] == ["4"]+++async def test_related_papers_builds_path_and_query():+ client = FakeAsyncHttpClient(+ FakeResponse(200, {"success": True, "results": [], "poolSize": 0, "truncated": False})+ )+ await aio_research.related_papers(+ client,+ "2105.05233",+ "diffusion image synthesis",+ mode="citers",+ k=20,+ rerank=False,+ anchor=["arxiv:2006.11239", "1503.03585"],+ )+ path, qs = split(client.last_path)+ assert path == "/v2/search/research/papers/2105.05233/similar"+ assert qs["intent"] == ["diffusion image synthesis"]+ assert qs["mode"] == ["citers"]+ assert qs["k"] == ["20"]+ assert qs["rerank"] == ["false"]+ assert qs["anchor"] == ["arxiv:2006.11239", "1503.03585"]+++async def test_related_papers_omits_rerank_when_none():+ client = FakeAsyncHttpClient(FakeResponse(200, {"success": True, "results": []}))+ await aio_research.related_papers(client, "1", "intent")+ _, qs = split(client.last_path)+ assert "rerank" not in qs+++async def test_search_github_builds_query_string():+ client = FakeAsyncHttpClient(FakeResponse(200, {"success": True, "results": []}))+ await aio_research.search_github(client, "milvus hybrid search", k=10)+ path, qs = split(client.last_path)+ assert path == "/v2/search/research/github"+ assert qs["query"] == ["milvus hybrid search"]+ assert qs["k"] == ["10"]+++async def test_error_mapping():+ client = FakeAsyncHttpClient(FakeResponse(400, {"error": "query is required"}))+ with pytest.raises(BadRequestError) as exc:+ await aio_research.search_papers(client, "q")+ assert "query is required" in str(exc.value)+ assert exc.value.status_code == 400++ client = FakeAsyncHttpClient(FakeResponse(404, {"error": "Not Found"}))+ with pytest.raises(FirecrawlError) as exc:+ await aio_research.inspect_paper(client, "pmid:0")+ assert exc.value.status_code == 404+++async def test_async_client_exposes_research_methods():+ from firecrawl.v2.client_async import AsyncFirecrawlClient++ for name in (+ "search_papers",+ "inspect_paper",+ "read_paper",+ "related_papers",+ "search_github",+ ):+ assert callable(getattr(AsyncFirecrawlClient, name)), name++ client = AsyncFirecrawlClient(api_key="fc-test")+ client.async_http_client = FakeAsyncHttpClient(+ FakeResponse(200, {"success": True, "results": []})+ )+ await client.search_papers("tau aggregation inhibitors", k=5)+ path, qs = split(client.async_http_client.last_path)+ assert path == "/v2/search/research/papers"+ assert qs["k"] == ["5"]+++async def test_async_research_methods_are_documented():+ """The docstrings are the disambiguation surface agents actually read."""+ from firecrawl.v2.client_async import AsyncFirecrawlClient++ doc = AsyncFirecrawlClient.search_papers.__doc__ or ""+ assert "PubMed" in doc+ assert 'categories=["research"]' in doc+++RESEARCH_METHODS = (+ "search_papers",+ "inspect_paper",+ "read_paper",+ "related_papers",+ "search_github",+)+++def _unified_client(response=None):+ """Top-level ``AsyncFirecrawl`` with its v2 HTTP layer mocked."""+ from firecrawl import AsyncFirecrawl++ client = AsyncFirecrawl(api_key="fc-test")+ client._v2_client.async_http_client = FakeAsyncHttpClient(+ response or FakeResponse(200, {"success": True})+ )+ return client+++def test_unified_async_client_exposes_research_methods():+ """``AsyncFirecrawl`` delegates method by method; pin all five."""+ from firecrawl import AsyncFirecrawl++ for name in RESEARCH_METHODS:+ assert callable(getattr(AsyncFirecrawl, name, None)), name+ assert (getattr(AsyncFirecrawl, name).__doc__ or "").strip(), name++ doc = AsyncFirecrawl.search_papers.__doc__ or ""+ assert "PubMed" in doc+ assert 'categories=["research"]' in doc+++async def test_unified_async_client_delegates_all_research_methods():+ client = _unified_client(FakeResponse(200, {"success": True, "results": []}))+ http = client._v2_client.async_http_client++ await client.search_papers("tau aggregation inhibitors", k=5)+ path, qs = split(http.last_path)+ assert path == "/v2/search/research/papers"+ assert qs["k"] == ["5"]++ await client.inspect_paper("pmid:12345678")+ path, _ = split(http.last_path)+ assert path == "/v2/search/research/papers/pmid%3A12345678"++ await client.read_paper("pmid:12345678", "primary endpoint", k=4)+ path, qs = split(http.last_path)+ assert path == "/v2/search/research/papers/pmid%3A12345678"+ assert qs["query"] == ["primary endpoint"]++ await client.related_papers("pmid:12345678", intent="replication attempts")+ path, qs = split(http.last_path)+ assert path == "/v2/search/research/papers/pmid%3A12345678/similar"+ assert qs["intent"] == ["replication attempts"]++ await client.search_github("pysam VCF parsing memory leak")+ path, qs = split(http.last_path)+ assert path == "/v2/search/research/github"+ assert qs["query"] == ["pysam VCF parsing memory leak"]+++async def test_unified_async_client_returns_response_verbatim():+ payload = {"success": True, "results": [{"paperId": "1", "primaryId": "pmid:1"}]}+ client = _unified_client(FakeResponse(200, payload))+ assert await client.search_papers("q") == payloaddiff --git a/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_research.py b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_research.pynew file mode 100644index 000000000..322a1c7ba--- /dev/null+++ b/apps/python-sdk/firecrawl/__tests__/unit/v2/methods/test_research.py@@ -0,0 +1,434 @@+"""+Unit tests for the v2 research paper index methods.++Mirrors the JS SDK suite in+``apps/js-sdk/firecrawl/src/__tests__/unit/v2/research.test.ts``: the HTTP+client is mocked and the tests pin the method names, the request paths and the+query-string construction. No network access.++These endpoints are the research paper index (``/v2/search/research``) β not the+``search(categories=["research"])`` website filter.+"""++from urllib.parse import parse_qs, urlparse++import pytest++from firecrawl.v2.methods import research+from firecrawl.v2.utils.error_handler import (+ BadRequestError,+ FirecrawlError,+)+++class FakeResponse:+ """Minimal stand-in for ``requests.Response``."""++ def __init__(self, status_code=200, payload=None):+ self.status_code = status_code+ self._payload = {} if payload is None else payload+ self.text = str(self._payload)++ def json(self):+ return self._payload+++class FakeHttpClient:+ """Records every path requested through ``get``."""++ def __init__(self, response=None):+ self.calls = []+ self._response = response or FakeResponse()++ def get(self, path):+ self.calls.append(path)+ return self._response++ @property+ def last_path(self):+ return self.calls[-1]+++def split(path):+ """Split a recorded path into (path, {param: [values]})."""+ parsed = urlparse(path)+ return parsed.path, parse_qs(parsed.query, keep_blank_values=True)+++class TestSearchPapers:+ """search_papers -> GET /v2/search/research/papers"""++ def test_builds_query_string_with_exploded_arrays(self):+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))+ research.search_papers(+ client,+ "diffusion models",+ k=10,+ authors=["Ho", "Abbeel"],+ categories=["cs.LG", "stat.ML"],+ from_date="2020-01-01",+ to_date="2024-12-31",+ )+ path, qs = split(client.last_path)+ assert path == "/v2/search/research/papers"+ assert qs["query"] == ["diffusion models"]+ assert qs["k"] == ["10"]+ # list values are exploded into repeated params, not comma-joined+ assert qs["authors"] == ["Ho", "Abbeel"]+ assert qs["categories"] == ["cs.LG", "stat.ML"]+ # from_date/to_date are sent as the wire names `from`/`to`+ assert qs["from"] == ["2020-01-01"]+ assert qs["to"] == ["2024-12-31"]++ def test_biomedical_query_is_passed_through_percent_encoded(self):+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))+ research.search_papers(+ client, "CRISPR base editing off-target effects in T cells"+ )+ assert "CRISPR%20base%20editing" in client.last_path+ _, qs = split(client.last_path)+ assert qs["query"] == ["CRISPR base editing off-target effects in T cells"]++ def test_omits_absent_options(self):+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))+ research.search_papers(client, "q")+ path, qs = split(client.last_path)+ assert path == "/v2/search/research/papers"+ assert sorted(qs.keys()) == ["origin", "query"]+ assert qs["origin"][0].startswith("python-sdk@")++ def test_returns_the_response_body_verbatim(self):+ payload = {+ "success": True,+ "results": [+ {+ "paperId": "1",+ "primaryId": "pmid:12345678",+ "title": "t",+ "abstract": "a",+ "score": 0.1,+ }+ ],+ }+ client = FakeHttpClient(FakeResponse(200, payload))+ assert research.search_papers(client, "q") == payload++ def test_response_keys_are_not_snake_case_normalized(self):+ """Research responses are returned raw; keys stay camelCase."""+ payload = {"success": True, "results": [{"paperId": "1", "primaryId": "doi:10.1/x"}]}+ client = FakeHttpClient(FakeResponse(200, payload))+ result = research.search_papers(client, "q")+ assert "paperId" in result["results"][0]+ assert "paper_id" not in result["results"][0]+++class TestInspectPaper:+ """inspect_paper -> GET /v2/search/research/papers/{id}"""++ @pytest.mark.parametrize(+ "paper_id, expected_path",+ [+ ("arxiv:2105.05233", "/v2/search/research/papers/arxiv%3A2105.05233"),+ ("pmid:12345678", "/v2/search/research/papers/pmid%3A12345678"),+ ("pmcid:PMC1234567", "/v2/search/research/papers/pmcid%3APMC1234567"),+ (+ "doi:10.1038/s41586-020-2649-2",+ "/v2/search/research/papers/doi%3A10.1038%2Fs41586-020-2649-2",+ ),+ ("123456789", "/v2/search/research/papers/123456789"),+ ],+ )+ def test_encodes_namespaced_ids_in_the_path(self, paper_id, expected_path):+ client = FakeHttpClient(FakeResponse(200, {"success": True, "paper": {}}))+ research.inspect_paper(client, paper_id)+ path, _ = split(client.last_path)+ assert path == expected_path++ def test_sends_origin_like_the_other_research_methods(self):+ """Detail mode sends ``origin`` too β same attribution as the other four."""+ client = FakeHttpClient(FakeResponse(200, {"success": True, "paper": {}}))+ research.inspect_paper(client, "pmid:12345678")+ _, qs = split(client.last_path)+ assert sorted(qs.keys()) == ["origin"]+ assert qs["origin"][0].startswith("python-sdk@")+++class TestReadPaper:+ """read_paper -> GET /v2/search/research/papers/{id}?query=..."""++ def test_adds_query_and_k(self):+ client = FakeHttpClient(+ FakeResponse(200, {"success": True, "paper": {}, "passages": []})+ )+ research.read_paper(+ client,+ "pmid:12345678",+ "what was the primary endpoint?",+ k=4,+ )+ path, qs = split(client.last_path)+ assert path == "/v2/search/research/papers/pmid%3A12345678"+ assert qs["query"] == ["what was the primary endpoint?"]+ assert qs["k"] == ["4"]+ assert qs["origin"][0].startswith("python-sdk@")++ def test_omits_k_when_absent(self):+ client = FakeHttpClient(FakeResponse(200, {"success": True, "paper": {}}))+ research.read_paper(client, "123", "noise schedule")+ _, qs = split(client.last_path)+ assert sorted(qs.keys()) == ["origin", "query"]+++class TestRelatedPapers:+ """related_papers -> GET /v2/search/research/papers/{id}/similar++ Same endpoint the JS SDK exposes as ``research.similarPapers()``.+ """++ def test_builds_path_and_query_with_repeated_anchors_and_rerank(self):+ client = FakeHttpClient(+ FakeResponse(200, {"success": True, "results": [], "poolSize": 0, "truncated": False})+ )+ research.related_papers(+ client,+ "2105.05233",+ "diffusion image synthesis",+ mode="citers",+ k=20,+ rerank=False,+ anchor=["arxiv:2006.11239", "1503.03585"],+ )+ path, qs = split(client.last_path)+ assert path == "/v2/search/research/papers/2105.05233/similar"+ assert qs["intent"] == ["diffusion image synthesis"]+ assert qs["mode"] == ["citers"]+ assert qs["k"] == ["20"]+ # booleans are lower-cased, not Python-cased+ assert qs["rerank"] == ["false"]+ assert qs["anchor"] == ["arxiv:2006.11239", "1503.03585"]++ def test_rerank_true_is_lowercased(self):+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))+ research.related_papers(client, "1", "intent", rerank=True)+ _, qs = split(client.last_path)+ assert qs["rerank"] == ["true"]++ def test_omits_rerank_when_none(self):+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))+ research.related_papers(client, "1", "intent")+ _, qs = split(client.last_path)+ assert "rerank" not in qs+ assert qs["intent"] == ["intent"]+++class TestSearchGithub:+ """search_github -> GET /v2/search/research/github"""++ def test_builds_query_string(self):+ client = FakeHttpClient(FakeResponse(200, {"success": True, "results": []}))+ research.search_github(client, "milvus hybrid search", k=10)+ path, qs = split(client.last_path)+ assert path == "/v2/search/research/github"+ assert qs["query"] == ["milvus hybrid search"]+ assert qs["k"] == ["10"]+ assert qs["origin"][0].startswith("python-sdk@")+++class TestErrorMapping:+ """Non-200 responses raise the shared v2 error types."""++ def test_400_raises_bad_request(self):+ client = FakeHttpClient(FakeResponse(400, {"error": "query is required"}))+ with pytest.raises(BadRequestError) as exc:+ research.search_papers(client, "q")+ assert "query is required" in str(exc.value)+ assert exc.value.status_code == 400++ def test_404_raises_firecrawl_error(self):+ client = FakeHttpClient(FakeResponse(404, {"error": "Not Found"}))+ with pytest.raises(FirecrawlError) as exc:+ research.inspect_paper(client, "pmid:0")+ assert exc.value.status_code == 404++ def test_error_action_is_labelled_research(self):+ client = FakeHttpClient(FakeResponse(500, {"error": "boom"}))+ with pytest.raises(FirecrawlError) as exc:+ research.search_github(client, "q")+ assert "research" in str(exc.value)+++class TestClientWiring:+ """Pin the public method names on the sync client."""++ def _client(self, response=None):+ from firecrawl.v2.client import FirecrawlClient++ client = FirecrawlClient(api_key="fc-test")+ client.http_client = FakeHttpClient(response or FakeResponse(200, {"success": True}))+ return client++ def test_client_exposes_research_methods(self):+ from firecrawl.v2.client import FirecrawlClient++ for name in (+ "search_papers",+ "inspect_paper",+ "read_paper",+ "related_papers",+ "search_github",+ ):+ assert callable(getattr(FirecrawlClient, name)), name++ def test_search_papers_routes_to_the_paper_index(self):+ client = self._client(FakeResponse(200, {"success": True, "results": []}))+ client.search_papers("tau aggregation inhibitors", k=5)+ path, qs = split(client.http_client.last_path)+ assert path == "/v2/search/research/papers"+ assert qs["k"] == ["5"]++ def test_read_paper_routes_to_the_paper_id_path(self):+ client = self._client(FakeResponse(200, {"success": True, "paper": {}}))+ client.read_paper("pmid:12345678", "primary endpoint", k=2)+ path, qs = split(client.http_client.last_path)+ assert path == "/v2/search/research/papers/pmid%3A12345678"+ assert qs["query"] == ["primary endpoint"]++ def test_related_papers_routes_to_similar(self):+ client = self._client(FakeResponse(200, {"success": True, "results": []}))+ client.related_papers("pmid:12345678", intent="replication attempts")+ path, _ = split(client.http_client.last_path)+ assert path == "/v2/search/research/papers/pmid%3A12345678/similar"++ def test_research_methods_are_documented(self):+ """The docstrings are the disambiguation surface agents actually read."""+ from firecrawl.v2.client import FirecrawlClient++ doc = FirecrawlClient.search_papers.__doc__ or ""+ assert "PubMed" in doc+ assert 'categories=["research"]' in doc+++RESEARCH_METHODS = (+ "search_papers",+ "inspect_paper",+ "read_paper",+ "related_papers",+ "search_github",+)+++class TestUnifiedClientWiring:+ """The top-level ``Firecrawl`` client is what the README instantiates.++ ``Firecrawl`` does not inherit from the v2 client (unlike the JS SDK, where+ ``Firecrawl extends FirecrawlClient``) β it delegates method by method. Any+ method missing from that list is simply absent from the documented surface,+ so pin all five here.+ """++ def _client(self, response=None):+ from firecrawl import Firecrawl++ client = Firecrawl(api_key="fc-test")+ client._v2_client.http_client = FakeHttpClient(+ response or FakeResponse(200, {"success": True})+ )+ return client++ def test_unified_client_exposes_research_methods(self):+ from firecrawl import Firecrawl++ for name in RESEARCH_METHODS:+ assert callable(getattr(Firecrawl, name, None)), name++ def test_search_papers_delegates_to_the_paper_index(self):+ client = self._client(FakeResponse(200, {"success": True, "results": []}))+ client.search_papers("tau aggregation inhibitors", k=5)+ path, qs = split(client._v2_client.http_client.last_path)+ assert path == "/v2/search/research/papers"+ assert qs["query"] == ["tau aggregation inhibitors"]+ assert qs["k"] == ["5"]++ def test_inspect_paper_delegates(self):+ client = self._client(FakeResponse(200, {"success": True, "paper": {}}))+ client.inspect_paper("pmid:12345678")+ path, _ = split(client._v2_client.http_client.last_path)+ assert path == "/v2/search/research/papers/pmid%3A12345678"++ def test_read_paper_delegates(self):+ client = self._client(FakeResponse(200, {"success": True, "passages": []}))+ client.read_paper("pmid:12345678", "primary endpoint", k=4)+ path, qs = split(client._v2_client.http_client.last_path)+ assert path == "/v2/search/research/papers/pmid%3A12345678"+ assert qs["query"] == ["primary endpoint"]+ assert qs["k"] == ["4"]++ def test_related_papers_delegates(self):+ client = self._client(FakeResponse(200, {"success": True, "results": []}))+ client.related_papers("pmid:12345678", intent="replication attempts", k=20)+ path, qs = split(client._v2_client.http_client.last_path)+ assert path == "/v2/search/research/papers/pmid%3A12345678/similar"+ assert qs["intent"] == ["replication attempts"]+ assert qs["k"] == ["20"]++ def test_search_github_delegates(self):+ client = self._client(FakeResponse(200, {"success": True, "results": []}))+ client.search_github("pysam VCF parsing memory leak", k=3)+ path, qs = split(client._v2_client.http_client.last_path)+ assert path == "/v2/search/research/github"+ assert qs["query"] == ["pysam VCF parsing memory leak"]++ def test_returns_the_v2_response_verbatim(self):+ payload = {"success": True, "results": [{"paperId": "1", "primaryId": "pmid:1"}]}+ client = self._client(FakeResponse(200, payload))+ assert client.search_papers("q") == payload++ def test_delegated_docstrings_carry_the_disambiguation(self):+ from firecrawl import Firecrawl++ doc = Firecrawl.search_papers.__doc__ or ""+ assert "PubMed" in doc+ assert 'categories=["research"]' in doc+ for name in RESEARCH_METHODS:+ assert (getattr(Firecrawl, name).__doc__ or "").strip(), name+++class TestDocstringSingleSourceOfTruth:+ """The five docstrings live once, in ``v2/methods/research_docs.py``.++ Six layers surface the same five methods. Each keeps a full docstring, but+ the prose is applied from the canonical constants, so a correction to a+ factual claim is a one-place edit. These tests fail if a layer forks its+ own copy again.+ """++ def _layers(self):+ from firecrawl import AsyncFirecrawl, Firecrawl+ from firecrawl.v2.client import FirecrawlClient+ from firecrawl.v2.client_async import AsyncFirecrawlClient+ from firecrawl.v2.methods import research as sync_research+ from firecrawl.v2.methods import research_docs as docs+ from firecrawl.v2.methods.aio import research as aio_research++ return docs, {+ "impl": (sync_research, ""),+ "impl_aio": (aio_research, "AIO_"),+ "v2_client": (FirecrawlClient, "CLIENT_"),+ "v2_client_async": (AsyncFirecrawlClient, "ASYNC_CLIENT_"),+ "unified": (Firecrawl, "CLIENT_"),+ "unified_async": (AsyncFirecrawl, "ASYNC_CLIENT_"),+ }++ def test_every_layer_uses_the_canonical_docstring(self):+ docs, layers = self._layers()+ for name in RESEARCH_METHODS:+ for label, (obj, prefix) in layers.items():+ canonical = getattr(docs, f"{prefix}{name.upper()}_DOC")+ assert getattr(obj, name).__doc__ == canonical, f"{label}.{name}"++ def test_no_unresolved_placeholders_ship_to_users(self):+ """A template placeholder leaking into ``help()`` would be user-visible."""+ docs, layers = self._layers()+ for name in RESEARCH_METHODS:+ for obj, _ in layers.values():+ assert "|" not in (getattr(obj, name).__doc__ or "")diff --git a/apps/python-sdk/firecrawl/client.py b/apps/python-sdk/firecrawl/client.pyindex c998d3869..eb148aed9 100644--- a/apps/python-sdk/firecrawl/client.py+++ b/apps/python-sdk/firecrawl/client.py@@ -26,6 +26,19 @@ import loggingfrom .v1 import V1FirecrawlApp, AsyncV1FirecrawlAppfrom .v2 import FirecrawlClient as V2FirecrawlClientfrom .v2.client_async import AsyncFirecrawlClient+from .v2.methods.research_docs import (+ ASYNC_CLIENT_INSPECT_PAPER_DOC,+ ASYNC_CLIENT_READ_PAPER_DOC,+ ASYNC_CLIENT_RELATED_PAPERS_DOC,+ ASYNC_CLIENT_SEARCH_GITHUB_DOC,+ ASYNC_CLIENT_SEARCH_PAPERS_DOC,+ CLIENT_INSPECT_PAPER_DOC,+ CLIENT_READ_PAPER_DOC,+ CLIENT_RELATED_PAPERS_DOC,+ CLIENT_SEARCH_GITHUB_DOC,+ CLIENT_SEARCH_PAPERS_DOC,+ doc,+)from .v2.types import Document, ParseOptions, ScrapeOptionslogger = logging.getLogger("firecrawl")@@ -331,7 +344,29 @@ class Firecrawl:content_type=content_type,options=options,)-++ # Research paper index (/v2/search/research) β delegates to the v2 client.+ @doc(CLIENT_SEARCH_PAPERS_DOC)+ def search_papers(self, query: str, **kwargs):+ return self._v2_client.search_papers(query, **kwargs)++ @doc(CLIENT_INSPECT_PAPER_DOC)+ def inspect_paper(self, paper_id: str):+ return self._v2_client.inspect_paper(paper_id)++ @doc(CLIENT_READ_PAPER_DOC)+ def read_paper(self, paper_id: str, query: str, **kwargs):+ return self._v2_client.read_paper(paper_id, query, **kwargs)++ @doc(CLIENT_RELATED_PAPERS_DOC)+ def related_papers(self, paper_id: str, intent: str, **kwargs):+ return self._v2_client.related_papers(paper_id, intent, **kwargs)++ @doc(CLIENT_SEARCH_GITHUB_DOC)+ def search_github(self, query: str, **kwargs):+ return self._v2_client.search_github(query, **kwargs)++class AsyncFirecrawl:"""Async unified Firecrawl client (v2 by default, v1 under ``.v1``)."""@@ -444,6 +479,28 @@ class AsyncFirecrawl:options=options,)+ # Research paper index (/v2/search/research) β delegates to the v2 client.+ @doc(ASYNC_CLIENT_SEARCH_PAPERS_DOC)+ async def search_papers(self, query: str, **kwargs):+ return await self._v2_client.search_papers(query, **kwargs)++ @doc(ASYNC_CLIENT_INSPECT_PAPER_DOC)+ async def inspect_paper(self, paper_id: str):+ return await self._v2_client.inspect_paper(paper_id)++ @doc(ASYNC_CLIENT_READ_PAPER_DOC)+ async def read_paper(self, paper_id: str, query: str, **kwargs):+ return await self._v2_client.read_paper(paper_id, query, **kwargs)++ @doc(ASYNC_CLIENT_RELATED_PAPERS_DOC)+ async def related_papers(self, paper_id: str, intent: str, **kwargs):+ return await self._v2_client.related_papers(paper_id, intent, **kwargs)++ @doc(ASYNC_CLIENT_SEARCH_GITHUB_DOC)+ async def search_github(self, query: str, **kwargs):+ return await self._v2_client.search_github(query, **kwargs)++# Export Firecrawl as an alias for FirecrawlAppFirecrawlApp = FirecrawlAsyncFirecrawlApp = AsyncFirecrawldiff --git a/apps/python-sdk/firecrawl/v1/client.py b/apps/python-sdk/firecrawl/v1/client.pyindex df4236af7..b6525ee0d 100644--- a/apps/python-sdk/firecrawl/v1/client.py+++ b/apps/python-sdk/firecrawl/v1/client.py@@ -2631,7 +2631,7 @@ class V1FirecrawlApp:Initiates a deep research operation on a given query and polls until completion... deprecated::- /v1/deep-research is deprecated. Use /v2/search instead.+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.Args:query (str): Research query or topic to investigate@@ -2660,7 +2660,7 @@ class V1FirecrawlApp:"""import warningswarnings.warn(- "/v1/deep-research is deprecated. Use /v2/search instead.",+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",DeprecationWarning,stacklevel=2,)@@ -2734,7 +2734,7 @@ class V1FirecrawlApp:Initiates an asynchronous deep research operation... deprecated::- /v1/deep-research is deprecated. Use /v2/search instead.+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.Args:query (str): Research query or topic to investigate@@ -2756,7 +2756,7 @@ class V1FirecrawlApp:"""import warningswarnings.warn(- "/v1/deep-research is deprecated. Use /v2/search instead.",+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",DeprecationWarning,stacklevel=2,)@@ -2805,7 +2805,7 @@ class V1FirecrawlApp:Check the status of a deep research operation... deprecated::- /v1/deep-research is deprecated. Use /v2/search instead.+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.Args:id (str): The ID of the deep research operation.@@ -2830,7 +2830,7 @@ class V1FirecrawlApp:"""import warningswarnings.warn(- "/v1/deep-research is deprecated. Use /v2/search instead.",+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",DeprecationWarning,stacklevel=2,)@@ -4994,7 +4994,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):Initiates a deep research operation on a given query and polls until completion... deprecated::- /v1/deep-research is deprecated. Use /v2/search instead.+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.Args:query (str): Research query or topic to investigate@@ -5023,7 +5023,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):"""import warningswarnings.warn(- "/v1/deep-research is deprecated. Use /v2/search instead.",+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",DeprecationWarning,stacklevel=2,)@@ -5097,7 +5097,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):Initiates an asynchronous deep research operation... deprecated::- /v1/deep-research is deprecated. Use /v2/search instead.+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.Args:query (str): Research query or topic to investigate@@ -5119,7 +5119,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):"""import warningswarnings.warn(- "/v1/deep-research is deprecated. Use /v2/search instead.",+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",DeprecationWarning,stacklevel=2,)@@ -5157,7 +5157,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):Check the status of a deep research operation... deprecated::- /v1/deep-research is deprecated. Use /v2/search instead.+ /v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.Args:id (str): The ID of the deep research operation.@@ -5182,7 +5182,7 @@ class AsyncV1FirecrawlApp(V1FirecrawlApp):"""import warningswarnings.warn(- "/v1/deep-research is deprecated. Use /v2/search instead.",+ "/v1/deep-research is deprecated. Use /v2/search for web research, or the v2 research paper index (search_papers()) for scientific literature.",DeprecationWarning,stacklevel=2,)diff --git a/apps/python-sdk/firecrawl/v2/client.py b/apps/python-sdk/firecrawl/v2/client.pyindex 64371d85b..e2cbf8094 100644--- a/apps/python-sdk/firecrawl/v2/client.py+++ b/apps/python-sdk/firecrawl/v2/client.py@@ -68,6 +68,14 @@ from .methods import agent as agent_modulefrom .methods import browser as browser_modulefrom .methods import monitor as monitor_modulefrom .methods import research as research_module+from .methods.research_docs import (+ CLIENT_INSPECT_PAPER_DOC,+ CLIENT_READ_PAPER_DOC,+ CLIENT_RELATED_PAPERS_DOC,+ CLIENT_SEARCH_GITHUB_DOC,+ CLIENT_SEARCH_PAPERS_DOC,+ doc,+)from .watcher import Watcher# Kwargs that map to ScrapeOptions fields. Used by async crawl normalization@@ -225,18 +233,24 @@ class FirecrawlClient:) if any(v is not None for v in [formats, headers, include_tags, exclude_tags, only_main_content, timeout, wait_for, mobile, parsers, actions, location, skip_tls_verification, remove_base64_images, fast_mode, use_mock, block_ads, proxy, max_age, store_in_cache, lockdown, threat_protection, profile, audit_metadata, integration]) else Nonereturn scrape_module.scrape(self.http_client, url, options)+ # Research paper index (/v2/search/research)+ @doc(CLIENT_SEARCH_PAPERS_DOC)def search_papers(self, query: str, **kwargs):return research_module.search_papers(self.http_client, query, **kwargs)+ @doc(CLIENT_INSPECT_PAPER_DOC)def inspect_paper(self, paper_id: str):return research_module.inspect_paper(self.http_client, paper_id)+ @doc(CLIENT_READ_PAPER_DOC)def read_paper(self, paper_id: str, query: str, **kwargs):return research_module.read_paper(self.http_client, paper_id, query, **kwargs)+ @doc(CLIENT_RELATED_PAPERS_DOC)def related_papers(self, paper_id: str, intent: str, **kwargs):return research_module.related_papers(self.http_client, paper_id, intent, **kwargs)+ @doc(CLIENT_SEARCH_GITHUB_DOC)def search_github(self, query: str, **kwargs):return research_module.search_github(self.http_client, query, **kwargs)diff --git a/apps/python-sdk/firecrawl/v2/client_async.py b/apps/python-sdk/firecrawl/v2/client_async.pyindex 4b45e6207..a9db3e2c8 100644--- a/apps/python-sdk/firecrawl/v2/client_async.py+++ b/apps/python-sdk/firecrawl/v2/client_async.py@@ -63,6 +63,14 @@ from .methods.aio import agent as async_agent # type: ignore[attr-defined]from .methods.aio import browser as async_browser # type: ignore[attr-defined]from .methods.aio import monitor as async_monitor # type: ignore[attr-defined]from .methods.aio import research as async_research # type: ignore[attr-defined]+from .methods.research_docs import (+ ASYNC_CLIENT_INSPECT_PAPER_DOC,+ ASYNC_CLIENT_READ_PAPER_DOC,+ ASYNC_CLIENT_RELATED_PAPERS_DOC,+ ASYNC_CLIENT_SEARCH_GITHUB_DOC,+ ASYNC_CLIENT_SEARCH_PAPERS_DOC,+ doc,+)from .client import _SCRAPE_OPTION_KEYSfrom .watcher_async import AsyncWatcher@@ -108,18 +116,24 @@ class AsyncFirecrawlClient:options = ScrapeOptions(**{k: v for k, v in kwargs.items() if v is not None}) if kwargs else Nonereturn await async_scrape.scrape(self.async_http_client, url, options)+ # Research paper index (/v2/search/research)+ @doc(ASYNC_CLIENT_SEARCH_PAPERS_DOC)async def search_papers(self, query: str, **kwargs):return await async_research.search_papers(self.async_http_client, query, **kwargs)+ @doc(ASYNC_CLIENT_INSPECT_PAPER_DOC)async def inspect_paper(self, paper_id: str):return await async_research.inspect_paper(self.async_http_client, paper_id)+ @doc(ASYNC_CLIENT_READ_PAPER_DOC)async def read_paper(self, paper_id: str, query: str, **kwargs):return await async_research.read_paper(self.async_http_client, paper_id, query, **kwargs)+ @doc(ASYNC_CLIENT_RELATED_PAPERS_DOC)async def related_papers(self, paper_id: str, intent: str, **kwargs):return await async_research.related_papers(self.async_http_client, paper_id, intent, **kwargs)+ @doc(ASYNC_CLIENT_SEARCH_GITHUB_DOC)async def search_github(self, query: str, **kwargs):return await async_research.search_github(self.async_http_client, query, **kwargs)diff --git a/apps/python-sdk/firecrawl/v2/methods/aio/research.py b/apps/python-sdk/firecrawl/v2/methods/aio/research.pyindex 07897c9ba..e8725987a 100644--- a/apps/python-sdk/firecrawl/v2/methods/aio/research.py+++ b/apps/python-sdk/firecrawl/v2/methods/aio/research.py@@ -1,5 +1,30 @@"""Async research functionality for Firecrawl v2 API.++These functions query Firecrawl's **research paper index** (~43M paper+abstracts) served at ``/v2/search/research``. The corpus is roughly 90%+biomedical and life sciences β PubMed, bioRxiv and medRxiv β with arXiv+covering physics, mathematics and computer science.++.. warning::+ This is **not** the same thing as ``search(categories=["research"])``.+ That option is a website/domain filter applied to ordinary web search: it+ restricts Google-style results to about 14 academic domains+ (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, sciencedirect.com, ...)+ and returns web page snippets. The functions in this module query the+ paper index itself and return ranked paper records with full abstracts,+ passage-level reads and citation-graph neighbours.++ Use ``search_papers()`` for literature search; use+ ``search(categories=["research"])`` when you want ordinary web results+ narrowed to academic sites.++.. note::+ **Response keys are camelCase.** Unlike the rest of the Python SDK, these+ functions return the raw JSON body from the API as a ``dict``: it is not+ parsed into typed models and it is **not** normalized to snake_case. Expect+ ``paperId``, ``primaryId``, ``createdDate``, ``updateDate``,+ ``articleRank``, ``seedOverlap``, ``poolSize`` and so on."""from typing import Any, Dict, List, Optional@@ -8,6 +33,14 @@ from urllib.parse import quotefrom ...utils import handle_response_errorfrom ...utils.http_client_async import AsyncHttpClientfrom ...utils.get_version import get_version+from ..research_docs import (+ AIO_INSPECT_PAPER_DOC,+ AIO_READ_PAPER_DOC,+ AIO_RELATED_PAPERS_DOC,+ AIO_SEARCH_GITHUB_DOC,+ AIO_SEARCH_PAPERS_DOC,+ doc,+)BASE = "/v2/search/research"@@ -33,6 +66,7 @@ async def _get(client: AsyncHttpClient, path: str) -> Dict[str, Any]:return response.json()+@doc(AIO_SEARCH_PAPERS_DOC)async def search_papers(client: AsyncHttpClient,query: str,@@ -61,10 +95,15 @@ async def search_papers()+@doc(AIO_INSPECT_PAPER_DOC)async def inspect_paper(client: AsyncHttpClient, paper_id: str) -> Dict[str, Any]:- return await _get(client, f"{BASE}/papers/{quote(paper_id, safe='')}")+ return await _get(+ client,+ f"{BASE}/papers/{quote(paper_id, safe='')}" + _query({"origin": ORIGIN}),+ )+@doc(AIO_READ_PAPER_DOC)async def read_paper(client: AsyncHttpClient,paper_id: str,@@ -79,6 +118,7 @@ async def read_paper()+@doc(AIO_RELATED_PAPERS_DOC)async def related_papers(client: AsyncHttpClient,paper_id: str,@@ -105,6 +145,7 @@ async def related_papers()+@doc(AIO_SEARCH_GITHUB_DOC)async def search_github(client: AsyncHttpClient,query: str,diff --git a/apps/python-sdk/firecrawl/v2/methods/research.py b/apps/python-sdk/firecrawl/v2/methods/research.pyindex 3255396ac..79d4d92cc 100644--- a/apps/python-sdk/firecrawl/v2/methods/research.py+++ b/apps/python-sdk/firecrawl/v2/methods/research.py@@ -1,5 +1,30 @@"""Research functionality for Firecrawl v2 API.++These functions query Firecrawl's **research paper index** (~43M paper+abstracts) served at ``/v2/search/research``. The corpus is roughly 90%+biomedical and life sciences β PubMed, bioRxiv and medRxiv β with arXiv+covering physics, mathematics and computer science.++.. warning::+ This is **not** the same thing as ``search(categories=["research"])``.+ That option is a website/domain filter applied to ordinary web search: it+ restricts Google-style results to about 14 academic domains+ (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, sciencedirect.com, ...)+ and returns web page snippets. The functions in this module query the+ paper index itself and return ranked paper records with full abstracts,+ passage-level reads and citation-graph neighbours.++ Use ``search_papers()`` for literature search; use+ ``search(categories=["research"])`` when you want ordinary web results+ narrowed to academic sites.++.. note::+ **Response keys are camelCase.** Unlike the rest of the Python SDK, these+ functions return the raw JSON body from the API as a ``dict``: it is not+ parsed into typed models and it is **not** normalized to snake_case. Expect+ ``paperId``, ``primaryId``, ``createdDate``, ``updateDate``,+ ``articleRank``, ``seedOverlap``, ``poolSize`` and so on."""from typing import Any, Dict, List, Optional@@ -7,6 +32,14 @@ from urllib.parse import quotefrom ..utils import HttpClient, handle_response_errorfrom ..utils.get_version import get_version+from .research_docs import (+ INSPECT_PAPER_DOC,+ READ_PAPER_DOC,+ RELATED_PAPERS_DOC,+ SEARCH_GITHUB_DOC,+ SEARCH_PAPERS_DOC,+ doc,+)BASE = "/v2/search/research"@@ -32,6 +65,7 @@ def _get(client: HttpClient, path: str) -> Dict[str, Any]:return response.json()+@doc(SEARCH_PAPERS_DOC)def search_papers(client: HttpClient,query: str,@@ -60,10 +94,15 @@ def search_papers()+@doc(INSPECT_PAPER_DOC)def inspect_paper(client: HttpClient, paper_id: str) -> Dict[str, Any]:- return _get(client, f"{BASE}/papers/{quote(paper_id, safe='')}")+ return _get(+ client,+ f"{BASE}/papers/{quote(paper_id, safe='')}" + _query({"origin": ORIGIN}),+ )+@doc(READ_PAPER_DOC)def read_paper(client: HttpClient,paper_id: str,@@ -78,6 +117,7 @@ def read_paper()+@doc(RELATED_PAPERS_DOC)def related_papers(client: HttpClient,paper_id: str,@@ -104,6 +144,7 @@ def related_papers()+@doc(SEARCH_GITHUB_DOC)def search_github(client: HttpClient,query: str,diff --git a/apps/python-sdk/firecrawl/v2/methods/research_docs.py b/apps/python-sdk/firecrawl/v2/methods/research_docs.pynew file mode 100644index 000000000..57a4d9c43--- /dev/null+++ b/apps/python-sdk/firecrawl/v2/methods/research_docs.py@@ -0,0 +1,315 @@+"""+Canonical docstrings for the research paper index (``/v2/search/research``).++Single source of truth for the prose that documents ``search_papers``,+``inspect_paper``, ``read_paper``, ``related_papers`` and ``search_github``.+The same five methods are surfaced at six layers β the implementation functions+in ``research.py`` and ``aio/research.py``, and the delegating methods on+``FirecrawlClient``, ``AsyncFirecrawlClient``, ``Firecrawl`` and+``AsyncFirecrawl``. Every layer keeps a full docstring (``help()`` on any of+them shows the complete text), but the text lives here once, so a correction to+a factual claim β corpus size and composition, the+``search(categories=["research"])`` disambiguation, wire names, Args/Returns β+is a one-place edit instead of a six-way edit that drifts.++Apply a constant with the :func:`doc` decorator::++ @doc(SEARCH_PAPERS_DOC)+ def search_papers(client, query, ...):+ ...++``*_DOC`` documents the module-level functions, which take an explicit+``client``; ``CLIENT_*_DOC`` documents the client methods, which forward+``**kwargs``. The ``AIO_*`` and ``ASYNC_CLIENT_*`` variants are the same text+with the async wording substituted, never a separate copy of the prose.+"""++from typing import Callable, TypeVar++F = TypeVar("F", bound=Callable)+++def doc(text: str) -> Callable[[F], F]:+ """Attach ``text`` as the docstring of the decorated function.++ The function itself is returned unchanged β this only sets ``__doc__``.+ """++ def decorate(fn: F) -> F:+ fn.__doc__ = text+ return fn++ return decorate+++# Placeholders that the sync/async variants below substitute. They are not+# prose, so a template can never accidentally ship with one unresolved.+_CLIENT_ARG = "|CLIENT_ARG|"+_AWAIT = "|AWAIT|"++_SYNC_CLIENT_ARG = "client: HTTP client."+_ASYNC_CLIENT_ARG = "client: Async HTTP client."+++# ---------------------------------------------------------------------------+# Implementation functions (``research.py`` / ``aio/research.py``)+# ---------------------------------------------------------------------------++_SEARCH_PAPERS_TEMPLATE = """+Search the research paper index by abstract relevance.++Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of the+corpus β biomedical and life sciences) plus arXiv (physics, mathematics,+computer science). Semantic search over abstracts, not keyword matching.++This is **not** ``search(categories=["research"])``. That option only+restricts ordinary web search to ~14 academic websites and returns page+snippets; this function searches the paper index and returns paper records.++Args:+ |CLIENT_ARG|+ query: Natural-language query, e.g. ``"CRISPR base editing off-target+ effects in primary human T cells"``.+ k: Maximum number of papers to return.+ authors: Filter by author name(s). Repeated per value.+ categories: Filter by arXiv-style subject categories (e.g. ``["q-bio.GN"]``).+ Note this is the *paper* category filter, unrelated to the+ ``categories`` argument of ``search()``.+ from_date: Inclusive lower bound on publication date (``YYYY-MM-DD``).+ to_date: Inclusive upper bound on publication date (``YYYY-MM-DD``).++Returns:+ Raw API ``dict`` with ``success`` and ``results``. Keys are camelCase+ and are **not** normalized to snake_case: each result carries+ ``paperId``, ``primaryId`` (e.g. ``pmid:<id>``, ``doi:<id>``,+ ``arxiv:<id>``), ``ids``, ``title``, ``abstract`` and ``score``.++Example:+ >>> res = search_papers(client, "tau aggregation inhibitors in Alzheimer's", k=10)+ >>> res["results"][0]["paperId"]++See Also:+ ``inspect_paper``, ``read_paper``, ``related_papers``.+"""++_INSPECT_PAPER_TEMPLATE = """+Fetch metadata for a single paper in the research paper index.++Resolves against the same ~43M-abstract corpus as ``search_papers``+(PubMed / bioRxiv / medRxiv / arXiv).++Args:+ |CLIENT_ARG|+ paper_id: A canonical ``paperId`` returned by ``search_papers``, or a+ namespaced id key such as ``pmid:<id>``, ``pmcid:<id>``,+ ``doi:<doi>`` or ``arxiv:<id>``. Bare arXiv ids and arXiv URLs are+ also accepted.++Returns:+ Raw API ``dict`` with ``success`` and ``paper``. Keys are camelCase and+ are **not** normalized to snake_case β expect ``paperId``, ``ids``,+ ``title``, ``abstract``, ``authors``, ``categories``, ``createdDate``,+ ``updateDate``.++See Also:+ ``read_paper`` to search inside the body of a paper.+"""++_READ_PAPER_TEMPLATE = """+Read inside a paper: return the passages of its body that best match a query.++Full-text passage retrieval over the research paper index (PubMed /+bioRxiv / medRxiv / arXiv). Use this to answer a specific question against+one paper instead of re-reading the whole document.++Args:+ |CLIENT_ARG|+ paper_id: Canonical ``paperId`` or a namespaced id key+ (``pmid:<id>``, ``pmcid:<id>``, ``doi:<doi>``, ``arxiv:<id>``).+ query: What to look for inside the paper, e.g. ``"primary endpoint and+ hazard ratio"``.+ k: Maximum number of passages to return.++Returns:+ Raw API ``dict`` with ``success``, ``paper``, ``paperId``, ``query`` and+ ``passages`` (each ``{"text": ..., "score": ...}``). Keys are camelCase+ and are **not** normalized to snake_case.+"""++_RELATED_PAPERS_TEMPLATE = """+Find papers related to a seed paper via the citation graph.++Walks citations/references within the research paper index (PubMed /+bioRxiv / medRxiv / arXiv) and re-ranks candidates against your stated+``intent``, so "related" means related *for your purpose*, not merely+co-cited.++Args:+ |CLIENT_ARG|+ paper_id: Seed paper β canonical ``paperId`` or a namespaced id key+ (``pmid:<id>``, ``pmcid:<id>``, ``doi:<doi>``, ``arxiv:<id>``).+ intent: Required. What you want the related papers *for*, e.g.+ ``"replication attempts in larger cohorts"``. Used to re-rank.+ mode: Traversal mode over the citation graph, e.g. ``"citers"``.+ k: Maximum number of papers to return.+ rerank: Whether to apply the intent reranker. Serialized as+ ``"true"``/``"false"``.+ anchor: Additional seed papers to anchor the neighbourhood on.++Returns:+ Raw API ``dict`` with ``success``, ``results``, ``poolSize``,+ ``truncated`` and optional ``note``. Keys are camelCase and are **not**+ normalized to snake_case; each result carries a ``signals`` object with+ ``structural``, ``semantic``, ``articleRank`` and ``seedOverlap``.++Note:+ This is the Python name for the endpoint the JS SDK exposes as+ ``research.similarPapers()``. Same endpoint+ (``/v2/search/research/papers/{id}/similar``), different method name.+"""++_SEARCH_GITHUB_TEMPLATE = """+Search the developer index: GitHub issue/PR history and repository readmes.++This is the code-and-discussion companion to ``search_papers`` and is served+by the same ``/v2/search/research`` surface. It searches indexed GitHub+history and readmes β it does **not** search the paper corpus, and it is not+the same as ``search(categories=["github"])`` (which is a ``site:github.com``+filter on ordinary web search).++Args:+ |CLIENT_ARG|+ query: Natural-language query, e.g. ``"pysam VCF parsing memory leak"``.+ k: Maximum number of results to return.++Returns:+ Raw API ``dict`` with ``success`` and ``results``. Keys are camelCase and+ are **not** normalized to snake_case β expect ``resultType``, ``repo``,+ ``url``, ``pageType``, ``number`` and a ``scoreBreakdown`` object.+"""++SEARCH_PAPERS_DOC = _SEARCH_PAPERS_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG)+AIO_SEARCH_PAPERS_DOC = _SEARCH_PAPERS_TEMPLATE.replace(_CLIENT_ARG, _ASYNC_CLIENT_ARG)+INSPECT_PAPER_DOC = _INSPECT_PAPER_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG)+AIO_INSPECT_PAPER_DOC = _INSPECT_PAPER_TEMPLATE.replace(_CLIENT_ARG, _ASYNC_CLIENT_ARG)+READ_PAPER_DOC = _READ_PAPER_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG)+AIO_READ_PAPER_DOC = _READ_PAPER_TEMPLATE.replace(_CLIENT_ARG, _ASYNC_CLIENT_ARG)+RELATED_PAPERS_DOC = _RELATED_PAPERS_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG)+AIO_RELATED_PAPERS_DOC = _RELATED_PAPERS_TEMPLATE.replace(_CLIENT_ARG, _ASYNC_CLIENT_ARG)+SEARCH_GITHUB_DOC = _SEARCH_GITHUB_TEMPLATE.replace(_CLIENT_ARG, _SYNC_CLIENT_ARG)+AIO_SEARCH_GITHUB_DOC = _SEARCH_GITHUB_TEMPLATE.replace(_CLIENT_ARG, _ASYNC_CLIENT_ARG)+++# ---------------------------------------------------------------------------+# Client methods (``FirecrawlClient``, ``AsyncFirecrawlClient``, ``Firecrawl``,+# ``AsyncFirecrawl``) β same prose, wrapper-shaped Args.+# ---------------------------------------------------------------------------++_CLIENT_SEARCH_PAPERS_TEMPLATE = """+Search the research paper index by abstract relevance.++Queries ~43M paper abstracts: PubMed, bioRxiv and medRxiv (about 90% of+the corpus β biomedical and life sciences) plus arXiv (physics,+mathematics, computer science). Use this for literature search.++Not to be confused with ``search(categories=["research"])``: that is a+website/domain filter on ordinary web search (it narrows Google-style+results to ~14 academic domains and returns page snippets). This method+searches the paper index itself and returns ranked paper records.++Args:+ query: Natural-language query, e.g. ``"CRISPR base editing+ off-target effects in primary human T cells"``.+ **kwargs: ``k``, ``authors``, ``categories``, ``from_date``,+ ``to_date``.++Returns:+ Raw API ``dict`` with ``success`` and ``results``. These research+ responses are **not** normalized to snake_case like the rest of the+ SDK β keys are camelCase (``paperId``, ``primaryId``, ...).++Example:+ >>> |AWAIT|firecrawl.search_papers("GLP-1 receptor agonists cardiovascular outcomes", k=10)+"""++_CLIENT_INSPECT_PAPER_TEMPLATE = """+Fetch metadata for one paper in the research paper index.++Args:+ paper_id: Canonical ``paperId`` from ``search_papers``, or a+ namespaced id key such as ``pmid:<id>``, ``pmcid:<id>``,+ ``doi:<doi>`` or ``arxiv:<id>``.++Returns:+ Raw API ``dict`` with ``success`` and ``paper``. Keys are camelCase,+ **not** snake_case-normalized (``paperId``, ``createdDate``, ...).+"""++_CLIENT_READ_PAPER_TEMPLATE = """+Read inside a paper: return the body passages that best match a query.++Full-text passage retrieval over the research paper index (PubMed /+bioRxiv / medRxiv / arXiv).++Args:+ paper_id: Canonical ``paperId`` or namespaced id key+ (``pmid:<id>``, ``pmcid:<id>``, ``doi:<doi>``, ``arxiv:<id>``).+ query: What to look for inside the paper, e.g. ``"primary endpoint+ and hazard ratio"``.+ **kwargs: ``k`` (max passages).++Returns:+ Raw API ``dict`` with ``success``, ``paper``, ``paperId``, ``query``+ and ``passages``. Keys are camelCase, **not** snake_case-normalized.+"""++_CLIENT_RELATED_PAPERS_TEMPLATE = """+Find papers related to a seed paper via the citation graph.++Candidates are re-ranked against ``intent``, so "related" means related+for your stated purpose rather than merely co-cited.++Args:+ paper_id: Seed paper β canonical ``paperId`` or namespaced id key+ (``pmid:<id>``, ``pmcid:<id>``, ``doi:<doi>``, ``arxiv:<id>``).+ intent: What you want the related papers for, e.g. ``"replication+ attempts in larger cohorts"``.+ **kwargs: ``mode``, ``k``, ``rerank``, ``anchor``.++Returns:+ Raw API ``dict`` with ``success``, ``results``, ``poolSize``,+ ``truncated`` and optional ``note``. Keys are camelCase, **not**+ snake_case-normalized (``articleRank``, ``seedOverlap``, ...).++Note:+ The JS SDK exposes this same endpoint as+ ``research.similarPapers()``.+"""++_CLIENT_SEARCH_GITHUB_TEMPLATE = """+Search the developer index: GitHub issue/PR history and repo readmes.++The code-and-discussion companion to ``search_papers``, served by the+same ``/v2/search/research`` surface. It does not search the paper+corpus, and it is not ``search(categories=["github"])`` (which is just a+``site:github.com`` filter on ordinary web search).++Args:+ query: Natural-language query, e.g. ``"pysam VCF parsing memory leak"``.+ **kwargs: ``k``.++Returns:+ Raw API ``dict`` with ``success`` and ``results``. Keys are+ camelCase, **not** snake_case-normalized.+"""++CLIENT_SEARCH_PAPERS_DOC = _CLIENT_SEARCH_PAPERS_TEMPLATE.replace(_AWAIT, "")+ASYNC_CLIENT_SEARCH_PAPERS_DOC = _CLIENT_SEARCH_PAPERS_TEMPLATE.replace(_AWAIT, "await ")+CLIENT_INSPECT_PAPER_DOC = _CLIENT_INSPECT_PAPER_TEMPLATE.replace(_AWAIT, "")+ASYNC_CLIENT_INSPECT_PAPER_DOC = _CLIENT_INSPECT_PAPER_TEMPLATE.replace(_AWAIT, "await ")+CLIENT_READ_PAPER_DOC = _CLIENT_READ_PAPER_TEMPLATE.replace(_AWAIT, "")+ASYNC_CLIENT_READ_PAPER_DOC = _CLIENT_READ_PAPER_TEMPLATE.replace(_AWAIT, "await ")+CLIENT_RELATED_PAPERS_DOC = _CLIENT_RELATED_PAPERS_TEMPLATE.replace(_AWAIT, "")+ASYNC_CLIENT_RELATED_PAPERS_DOC = _CLIENT_RELATED_PAPERS_TEMPLATE.replace(_AWAIT, "await ")+CLIENT_SEARCH_GITHUB_DOC = _CLIENT_SEARCH_GITHUB_TEMPLATE.replace(_AWAIT, "")+ASYNC_CLIENT_SEARCH_GITHUB_DOC = _CLIENT_SEARCH_GITHUB_TEMPLATE.replace(_AWAIT, "await ")diff --git a/apps/python-sdk/firecrawl/v2/types.py b/apps/python-sdk/firecrawl/v2/types.pyindex 530044981..8039d3395 100644--- a/apps/python-sdk/firecrawl/v2/types.py+++ b/apps/python-sdk/firecrawl/v2/types.py@@ -559,12 +559,30 @@ SourceOption = Union[str, Source]class Category(BaseModel):"""Configuration for a search category.+ Categories narrow ordinary **web search**. They do not switch `search()` to+ a different index.+Supported categories:- - "github": Filter results to GitHub repositories- - "research": Filter results to research papers and academic sites+ - "github": Restrict web results to github.com (a `site:` filter)+ - "research": Restrict web results to a fixed list of ~14 academic+ *websites* (arxiv.org, pubmed.ncbi.nlm.nih.gov, nature.com, science.org,+ ieee.org, sciencedirect.com, biorxiv.org, medrxiv.org, ...). This is a+ website/domain filter and it returns ordinary web page results for those+ domains β **not** paper records.- "pdf": Filter results to PDF files (adds filetype:pdf to search)- "developer": Add developer results (issues, pull requests, READMEs anddocumentation) under `.developer`++ .. warning::+ ``categories=["research"]`` is **not** Firecrawl's research paper index.+ To search papers themselves β ~43M abstracts, about 90% biomedical+ (PubMed, bioRxiv, medRxiv) plus arXiv β with full abstracts, in-body+ passage reads and citation-graph expansion, use+ :meth:`Firecrawl.search_papers` (and ``inspect_paper``, ``read_paper``,+ ``related_papers``), which call ``/v2/search/research``.++ Rule of thumb: literature search β ``search_papers()``; web pages that+ happen to live on academic domains β ``search(categories=["research"])``."""type: strdiff --git a/apps/python-sdk/pyproject.toml b/apps/python-sdk/pyproject.tomlindex 8c9925118..9e3017afe 100644--- a/apps/python-sdk/pyproject.toml+++ b/apps/python-sdk/pyproject.toml@@ -5,7 +5,7 @@ build-backend = "setuptools.build_meta"[project]dynamic = ["version"]name = "firecrawl-py"-description = "Python SDK for Firecrawl API"+description = "Python SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts"readme = {file="README.md", content-type = "text/markdown"}requires-python = ">=3.8"dependencies = [@@ -41,9 +41,33 @@ classifiers = ["Topic :: Software Development :: Libraries :: Python Modules","Topic :: Text Processing","Topic :: Text Processing :: Indexing",+ "Intended Audience :: Science/Research",+ "Topic :: Scientific/Engineering",+ "Topic :: Scientific/Engineering :: Bio-Informatics",+ "Topic :: Scientific/Engineering :: Medical Science Apps.",]-keywords = ["SDK", "API", "firecrawl"]+keywords = [+ "SDK",+ "API",+ "firecrawl",+ "web scraping",+ "crawler",+ "web search",+ "literature search",+ "research",+ "scientific papers",+ "academic search",+ "biomedical",+ "life sciences",+ "pubmed",+ "biorxiv",+ "medrxiv",+ "arxiv",+ "preprints",+ "citations",+ "bioinformatics",+][project.urls]"Documentation" = "https://docs.firecrawl.dev"diff --git a/apps/python-sdk/setup.py b/apps/python-sdk/setup.pyindex 79f3922b4..c705e8c51 100644--- a/apps/python-sdk/setup.py+++ b/apps/python-sdk/setup.py@@ -22,7 +22,7 @@ setup(url="https://github.com/firecrawl/firecrawl",author="Mendable.ai",author_email="[email protected]",- description="Python SDK for Firecrawl API",+ description="Python SDK for the Firecrawl API: web scraping, crawling, web search, and scientific literature search over a research paper index of PubMed, bioRxiv, medRxiv and arXiv abstracts",long_description=long_description_content,long_description_content_type="text/markdown",packages=find_packages(),@@ -57,8 +57,16 @@ setup("Topic :: Software Development :: Libraries :: Python Modules","Topic :: Text Processing","Topic :: Text Processing :: Indexing",+ "Intended Audience :: Science/Research",+ "Topic :: Scientific/Engineering",+ "Topic :: Scientific/Engineering :: Bio-Informatics",+ "Topic :: Scientific/Engineering :: Medical Science Apps.",],- keywords="SDK API firecrawl",+ keywords=(+ "SDK API firecrawl web scraping crawler web search literature search "+ "research scientific papers academic search biomedical life sciences "+ "pubmed biorxiv medrxiv arxiv preprints citations bioinformatics"+ ),project_urls={"Documentation": "https://docs.firecrawl.dev","Source": "https://github.com/firecrawl/firecrawl",