Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
warn that the research search category moves to the Research Index on 2026-11-16 (#4550)
Kris Oei96426f20c951
diff --git a/apps/api/src/__tests__/routes/search-research-category-notice.routes.test.ts b/apps/api/src/__tests__/routes/search-research-category-notice.routes.test.tsnew file mode 100644index 000000000..b85422969--- /dev/null+++ b/apps/api/src/__tests__/routes/search-research-category-notice.routes.test.ts@@ -0,0 +1,56 @@+import express from "express";+import request from "supertest";+import { researchCategoryNoticeMiddleware } from "../../controllers/v2/search";++// The snips cover the live route; this pins the shapes of `categories` the+// middleware must and must not react to, which need no search provider.+const app = express();+app.use(express.json());+app.post("/v2/search", researchCategoryNoticeMiddleware, (_req, res) => {+ res.status(200).json({ success: true, data: {}, creditsUsed: 0, id: "x" });+});++async function post(categories: unknown) {+ return request(app).post("/v2/search").send({ query: "x", categories });+}++describe("research category notice", () => {+ it("sets the header and the body warning, nothing else", async () => {+ const res = await post(["research"]);++ expect(res.statusCode).toBe(200);+ expect(res.headers["warning"]).toMatch(/^299 - "/);+ expect(res.headers["warning"]).toContain("2026-11-16");+ expect(res.headers["deprecation"]).toBeUndefined();+ expect(res.headers["sunset"]).toBeUndefined();+ expect(res.headers["link"]).toBe(+ '<https://docs.firecrawl.dev/features/research>; rel="help"',+ );+ expect(res.body.warnings).toHaveLength(1);+ expect(res.body.warnings[0]).toContain("data.research");+ expect(res.body.replacement).toBeUndefined();+ });++ it.each([+ [[{ type: "research" }]],+ [["research", "pdf"]],+ [["github", { type: "research" }]],+ ])("fires for %j", async categories => {+ const res = await post(categories);+ expect(res.body.warnings).toHaveLength(1);+ });++ it.each([+ [undefined],+ [[]],+ [["github"]],+ [["pdf"]],+ [["developer"]],+ ["research"],+ [[null, 7]],+ ])("stays silent for %j", async categories => {+ const res = await post(categories);+ expect(res.headers["warning"]).toBeUndefined();+ expect(res.body).not.toHaveProperty("warnings");+ });+});diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.tsindex 5eebd963b..3c00ee095 100644--- a/apps/api/src/__tests__/snips/v2/search.test.ts+++ b/apps/api/src/__tests__/snips/v2/search.test.ts@@ -5,7 +5,7 @@ import {HAS_SEARCH,TEST_PRODUCTION,} from "../lib";-import { search, searchWithFailure, idmux, Identity } from "./lib";+import { search, searchRaw, searchWithFailure, idmux, Identity } from "./lib";import { config } from "../../../config";let identity: Identity;@@ -343,3 +343,68 @@ describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)("Search tests", () => {60000,);});++describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)(+ "research category notice",+ () => {+ it.concurrent(+ "carries the notice on every response that uses the category",+ async () => {+ const raw = await searchRaw(+ { query: "firecrawl", categories: ["research"] },+ identity,+ );+ expect(raw.statusCode).toBe(200);+ expect(raw.headers["warning"]).toMatch(/^299 - "/);+ expect(raw.headers["link"]).toContain(+ "https://docs.firecrawl.dev/features/research",+ );+ expect(raw.body.warnings).toHaveLength(1);+ expect(raw.body.warnings[0]).toContain("2026-11-16");+ },+ 60000,+ );++ it.concurrent(+ "reaches requests the schema rejects",+ async () => {+ const raw = await searchRaw(+ { query: "firecrawl", categories: ["research"], limit: 0 },+ identity,+ );+ expect(raw.statusCode).toBe(400);+ expect(raw.body.success).toBe(false);+ expect(raw.body.warnings).toHaveLength(1);+ },+ 60000,+ );++ concurrentIf(TEST_PRODUCTION)(+ "reaches requests auth rejects",+ async () => {+ const raw = await searchRaw(+ { query: "firecrawl", categories: ["research"] },+ { apiKey: "invalid-api-key", teamId: "" },+ );+ expect(raw.statusCode).toBe(401);+ expect(raw.headers["warning"]).toContain("2026-11-16");+ expect(raw.body.warnings).toHaveLength(1);+ },+ 60000,+ );++ it.concurrent(+ "stays silent for every other category",+ async () => {+ const raw = await searchRaw(+ { query: "firecrawl", categories: ["github"] },+ identity,+ );+ expect(raw.statusCode).toBe(200);+ expect(raw.headers["warning"]).toBeUndefined();+ expect(raw.body.warnings).toBeUndefined();+ },+ 60000,+ );+ },+);diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex b115cae2a..41d92d925 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -1,4 +1,4 @@-import { Response } from "express";+import { NextFunction, Request, Response } from "express";import { externalRequestId } from "../../lib/external-request-id";import { config } from "../../config";import {@@ -23,7 +23,7 @@ import {import { logger as _logger } from "../../lib/logger";import { ScrapeJobTimeoutError } from "../../lib/error";import { z } from "zod";-import { CategoryOption } from "../../lib/search-query-builder";+import { CategoryOption, hasCategory } from "../../lib/search-query-builder";import { executeSearch } from "../../search/execute";import type { BillingMetadata } from "../../services/billing/types";import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";@@ -46,6 +46,25 @@ import {import { wantsDeveloperCategory } from "../../search/developer";import { requestOrigin } from "../../lib/request-origin";import { isAgentInteropSecretValid } from "../../lib/agent-interop";+import { applyNotice, type Notice } from "../../lib/deprecations";++const RESEARCH_CATEGORY_NOTICE: Notice = {+ message:+ "On 2026-11-16, the 'research' search category will query the Firecrawl Research Index (PubMed, bioRxiv, medRxiv, arXiv) rather than restricting web results to a fixed list of 14 academic domains. Results will move from data.web to data.research and will match the records returned by the Research Index endpoint GET /search/research/papers, with the fields paperId, primaryId, ids, title, abstract and score. To adopt those records today, call GET /search/research/papers (https://docs.firecrawl.dev/api-reference/endpoint/research-search-papers). To continue receiving web pages from academic domains, use includeDomains. The github, pdf and developer categories are unchanged. See https://docs.firecrawl.dev/features/research",+ links: ['<https://docs.firecrawl.dev/features/research>; rel="help"'],+};++// Ahead of auth and validation so rejected requests carry the notice too.+export function researchCategoryNoticeMiddleware(+ req: Request,+ res: Response,+ next: NextFunction,+) {+ if (hasCategory(req.body?.categories, "research")) {+ applyNotice(res, RESEARCH_CATEGORY_NOTICE);+ }+ next();+}export async function searchController(req: RequestWithAuth<{}, SearchResponse, SearchRequest>,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 3f4c717e0..26bda62ca 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1,4 +1,5 @@import { Request, Response } from "express";+import { hasCategory } from "../../lib/search-query-builder";import { config } from "../../config";import { z } from "zod";import { protocolIncluded, checkUrl } from "../../lib/validateUrl";@@ -2422,12 +2423,7 @@ export const searchRequestSchema = z).refine(x => {const categories = x.categories ?? [];- const hasDeveloper = categories.some(category =>- typeof category === "string"- ? category === "developer"- : category.type === "developer",- );- return !hasDeveloper || categories.length === 1;+ return !hasCategory(categories, "developer") || categories.length === 1;}, "the developer category cannot be combined with other categories").refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts).transform(x => {diff --git a/apps/api/src/lib/deprecations.ts b/apps/api/src/lib/deprecations.tsindex 752599543..5cb4489f3 100644--- a/apps/api/src/lib/deprecations.ts+++ b/apps/api/src/lib/deprecations.ts@@ -9,6 +9,12 @@ interface Deprecation {docs?: string;}+export interface Notice {+ message: string;+ links?: string[];+ replacement?: string;+}+// Every legacy entry shipped together in #3469, so they share one date.const DEPRECATIONS = {v1_extract: {@@ -97,6 +103,29 @@ function quoteWarningText(s: string): string {return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;}+// Copies the body: the research controllers log the same object after responding.+export function applyNotice(res: Response, notice: Notice) {+ if (notice.links?.length) res.setHeader("Link", notice.links.join(", "));+ // RFC 7234 Warning header, code 299 = "Miscellaneous Persistent Warning".+ res.setHeader("Warning", `299 - ${quoteWarningText(notice.message)}`);++ const originalJson = res.json.bind(res);+ res.json = (body: any) => {+ if (body && typeof body === "object" && !Array.isArray(body)) {+ const existing = Array.isArray(body.warnings) ? body.warnings : [];+ const annotated: any = {+ ...body,+ warnings: [...existing, notice.message],+ };+ if (notice.replacement && annotated.replacement === undefined) {+ annotated.replacement = notice.replacement;+ }+ return originalJson(annotated);+ }+ return originalJson(body);+ };+}+export function deprecationMiddleware(key: DeprecationKey) {const dep: Deprecation = DEPRECATIONS[key];return (req: Request, res: Response, next: NextFunction) => {@@ -112,27 +141,11 @@ export function deprecationMiddleware(key: DeprecationKey) {if (dep.replacement) {links.push(`<${dep.replacement}>; rel="successor-version"`);}- if (links.length > 0) res.setHeader("Link", links.join(", "));-- // RFC 7234 Warning header, code 299 = "Miscellaneous Persistent Warning".- res.setHeader("Warning", `299 - ${quoteWarningText(dep.message)}`);-- const originalJson = res.json.bind(res);- res.json = (body: any) => {- if (body && typeof body === "object" && !Array.isArray(body)) {- // Copy: the research controllers log the same object after responding.- const existing = Array.isArray(body.warnings) ? body.warnings : [];- const annotated: any = {- ...body,- warnings: [...existing, dep.message],- };- if (dep.replacement && annotated.replacement === undefined) {- annotated.replacement = dep.replacement;- }- return originalJson(annotated);- }- return originalJson(body);- };+ applyNotice(res, {+ message: dep.message,+ links,+ replacement: dep.replacement,+ });next();};}diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.tsindex 0102db692..bcbd6d5aa 100644--- a/apps/api/src/lib/search-query-builder.ts+++ b/apps/api/src/lib/search-query-builder.ts@@ -38,6 +38,16 @@ const DEFAULT_RESEARCH_SITES = ["medrxiv.org",];+export function hasCategory(+ categories: unknown,+ type: CategoryInput["type"],+): boolean {+ return (+ Array.isArray(categories) &&+ categories.some(c => (typeof c === "string" ? c : c?.type) === type)+ );+}+/*** Builds a search query with category filters* @param baseQuery The base search querydiff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex c6f9f1646..157c798ca 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -5,7 +5,10 @@ import { RateLimiterMode } from "../types";import { registerMcpActionLogReadRoute } from "./mcp-action-logs";import { SEARCH_CREDITS_FEATURE_ID } from "../services/autumn/autumn.service";import expressWs from "express-ws";-import { searchController } from "../controllers/v2/search";+import {+ researchCategoryNoticeMiddleware,+ searchController,+} from "../controllers/v2/search";import { feedbackController } from "../controllers/v2/feedback/controller";import { searchFeedbackController } from "../controllers/v2/search-feedback";import { scrapeController } from "../controllers/v2/scrape";@@ -185,6 +188,7 @@ registerMcpActionLogReadRoute(v2Router.post("/search",+ researchCategoryNoticeMiddleware,authMiddleware(RateLimiterMode.Search, { allowKeyless: true }),countryCheck,checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),diff --git a/apps/api/src/search/developer.ts b/apps/api/src/search/developer.tsindex 78e5c6e51..2304ccd88 100644--- a/apps/api/src/search/developer.ts+++ b/apps/api/src/search/developer.ts@@ -1,16 +1,12 @@import type { Logger } from "winston";import type { WebSearchResult } from "../lib/entities";import { fetchResearchUpstream } from "../lib/research-upstream";-import type { CategoryOption } from "../lib/search-query-builder";+import { hasCategory, type CategoryOption } from "../lib/search-query-builder";const DEVELOPER_QUERY_KEYS = ["query", "k"];export function wantsDeveloperCategory(categories?: CategoryOption[]): boolean {- return (categories ?? []).some(category =>- typeof category === "string"- ? category === "developer"- : category.type === "developer",- );+ return hasCategory(categories, "developer");}export async function searchDeveloperCategory(