Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

warn that the research search category moves to the Research Index on 2026-11-16 (#4550)

Kris Oei96426f20c951
diff --git a/apps/api/src/__tests__/routes/search-research-category-notice.routes.test.ts b/apps/api/src/__tests__/routes/search-research-category-notice.routes.test.ts
new file mode 100644
index 000000000..b85422969
--- /dev/null
+++ b/apps/api/src/__tests__/routes/search-research-category-notice.routes.test.ts
@@ -0,0 +1,56 @@
+import express from "express";
+import request from "supertest";
+import { researchCategoryNoticeMiddleware } from "../../controllers/v2/search";
+
+// The snips cover the live route; this pins the shapes of `categories` the
+// middleware must and must not react to, which need no search provider.
+const app = express();
+app.use(express.json());
+app.post("/v2/search", researchCategoryNoticeMiddleware, (_req, res) => {
+ res.status(200).json({ success: true, data: {}, creditsUsed: 0, id: "x" });
+});
+
+async function post(categories: unknown) {
+ return request(app).post("/v2/search").send({ query: "x", categories });
+}
+
+describe("research category notice", () => {
+ it("sets the header and the body warning, nothing else", async () => {
+ const res = await post(["research"]);
+
+ expect(res.statusCode).toBe(200);
+ expect(res.headers["warning"]).toMatch(/^299 - "/);
+ expect(res.headers["warning"]).toContain("2026-11-16");
+ expect(res.headers["deprecation"]).toBeUndefined();
+ expect(res.headers["sunset"]).toBeUndefined();
+ expect(res.headers["link"]).toBe(
+ '<https://docs.firecrawl.dev/features/research>; rel="help"',
+ );
+ expect(res.body.warnings).toHaveLength(1);
+ expect(res.body.warnings[0]).toContain("data.research");
+ expect(res.body.replacement).toBeUndefined();
+ });
+
+ it.each([
+ [[{ type: "research" }]],
+ [["research", "pdf"]],
+ [["github", { type: "research" }]],
+ ])("fires for %j", async categories => {
+ const res = await post(categories);
+ expect(res.body.warnings).toHaveLength(1);
+ });
+
+ it.each([
+ [undefined],
+ [[]],
+ [["github"]],
+ [["pdf"]],
+ [["developer"]],
+ ["research"],
+ [[null, 7]],
+ ])("stays silent for %j", async categories => {
+ const res = await post(categories);
+ expect(res.headers["warning"]).toBeUndefined();
+ expect(res.body).not.toHaveProperty("warnings");
+ });
+});
diff --git a/apps/api/src/__tests__/snips/v2/search.test.ts b/apps/api/src/__tests__/snips/v2/search.test.ts
index 5eebd963b..3c00ee095 100644
--- a/apps/api/src/__tests__/snips/v2/search.test.ts
+++ b/apps/api/src/__tests__/snips/v2/search.test.ts
@@ -5,7 +5,7 @@ import {
HAS_SEARCH,
TEST_PRODUCTION,
} from "../lib";
-import { search, searchWithFailure, idmux, Identity } from "./lib";
+import { search, searchRaw, searchWithFailure, idmux, Identity } from "./lib";
import { config } from "../../../config";
let identity: Identity;
@@ -343,3 +343,68 @@ describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)("Search tests", () => {
60000,
);
});
+
+describeIf(TEST_PRODUCTION || HAS_SEARCH || HAS_PROXY)(
+ "research category notice",
+ () => {
+ it.concurrent(
+ "carries the notice on every response that uses the category",
+ async () => {
+ const raw = await searchRaw(
+ { query: "firecrawl", categories: ["research"] },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+ expect(raw.headers["warning"]).toMatch(/^299 - "/);
+ expect(raw.headers["link"]).toContain(
+ "https://docs.firecrawl.dev/features/research",
+ );
+ expect(raw.body.warnings).toHaveLength(1);
+ expect(raw.body.warnings[0]).toContain("2026-11-16");
+ },
+ 60000,
+ );
+
+ it.concurrent(
+ "reaches requests the schema rejects",
+ async () => {
+ const raw = await searchRaw(
+ { query: "firecrawl", categories: ["research"], limit: 0 },
+ identity,
+ );
+ expect(raw.statusCode).toBe(400);
+ expect(raw.body.success).toBe(false);
+ expect(raw.body.warnings).toHaveLength(1);
+ },
+ 60000,
+ );
+
+ concurrentIf(TEST_PRODUCTION)(
+ "reaches requests auth rejects",
+ async () => {
+ const raw = await searchRaw(
+ { query: "firecrawl", categories: ["research"] },
+ { apiKey: "invalid-api-key", teamId: "" },
+ );
+ expect(raw.statusCode).toBe(401);
+ expect(raw.headers["warning"]).toContain("2026-11-16");
+ expect(raw.body.warnings).toHaveLength(1);
+ },
+ 60000,
+ );
+
+ it.concurrent(
+ "stays silent for every other category",
+ async () => {
+ const raw = await searchRaw(
+ { query: "firecrawl", categories: ["github"] },
+ identity,
+ );
+ expect(raw.statusCode).toBe(200);
+ expect(raw.headers["warning"]).toBeUndefined();
+ expect(raw.body.warnings).toBeUndefined();
+ },
+ 60000,
+ );
+ },
+);
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index b115cae2a..41d92d925 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -1,4 +1,4 @@
-import { Response } from "express";
+import { NextFunction, Request, Response } from "express";
import { externalRequestId } from "../../lib/external-request-id";
import { config } from "../../config";
import {
@@ -23,7 +23,7 @@ import {
import { logger as _logger } from "../../lib/logger";
import { ScrapeJobTimeoutError } from "../../lib/error";
import { z } from "zod";
-import { CategoryOption } from "../../lib/search-query-builder";
+import { CategoryOption, hasCategory } from "../../lib/search-query-builder";
import { executeSearch } from "../../search/execute";
import type { BillingMetadata } from "../../services/billing/types";
import { getSearchForcedKind, getSearchZDR } from "../../lib/zdr-helpers";
@@ -46,6 +46,25 @@ import {
import { wantsDeveloperCategory } from "../../search/developer";
import { requestOrigin } from "../../lib/request-origin";
import { isAgentInteropSecretValid } from "../../lib/agent-interop";
+import { applyNotice, type Notice } from "../../lib/deprecations";
+
+const RESEARCH_CATEGORY_NOTICE: Notice = {
+ message:
+ "On 2026-11-16, the 'research' search category will query the Firecrawl Research Index (PubMed, bioRxiv, medRxiv, arXiv) rather than restricting web results to a fixed list of 14 academic domains. Results will move from data.web to data.research and will match the records returned by the Research Index endpoint GET /search/research/papers, with the fields paperId, primaryId, ids, title, abstract and score. To adopt those records today, call GET /search/research/papers (https://docs.firecrawl.dev/api-reference/endpoint/research-search-papers). To continue receiving web pages from academic domains, use includeDomains. The github, pdf and developer categories are unchanged. See https://docs.firecrawl.dev/features/research",
+ links: ['<https://docs.firecrawl.dev/features/research>; rel="help"'],
+};
+
+// Ahead of auth and validation so rejected requests carry the notice too.
+export function researchCategoryNoticeMiddleware(
+ req: Request,
+ res: Response,
+ next: NextFunction,
+) {
+ if (hasCategory(req.body?.categories, "research")) {
+ applyNotice(res, RESEARCH_CATEGORY_NOTICE);
+ }
+ next();
+}
export async function searchController(
req: RequestWithAuth<{}, SearchResponse, SearchRequest>,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 3f4c717e0..26bda62ca 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1,4 +1,5 @@
import { Request, Response } from "express";
+import { hasCategory } from "../../lib/search-query-builder";
import { config } from "../../config";
import { z } from "zod";
import { protocolIncluded, checkUrl } from "../../lib/validateUrl";
@@ -2422,12 +2423,7 @@ export const searchRequestSchema = z
)
.refine(x => {
const categories = x.categories ?? [];
- const hasDeveloper = categories.some(category =>
- typeof category === "string"
- ? category === "developer"
- : category.type === "developer",
- );
- return !hasDeveloper || categories.length === 1;
+ return !hasCategory(categories, "developer") || categories.length === 1;
}, "the developer category cannot be combined with other categories")
.refine(x => waitForRefine(x.scrapeOptions), waitForRefineOpts)
.transform(x => {
diff --git a/apps/api/src/lib/deprecations.ts b/apps/api/src/lib/deprecations.ts
index 752599543..5cb4489f3 100644
--- a/apps/api/src/lib/deprecations.ts
+++ b/apps/api/src/lib/deprecations.ts
@@ -9,6 +9,12 @@ interface Deprecation {
docs?: string;
}
+export interface Notice {
+ message: string;
+ links?: string[];
+ replacement?: string;
+}
+
// Every legacy entry shipped together in #3469, so they share one date.
const DEPRECATIONS = {
v1_extract: {
@@ -97,6 +103,29 @@ function quoteWarningText(s: string): string {
return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;
}
+// Copies the body: the research controllers log the same object after responding.
+export function applyNotice(res: Response, notice: Notice) {
+ if (notice.links?.length) res.setHeader("Link", notice.links.join(", "));
+ // RFC 7234 Warning header, code 299 = "Miscellaneous Persistent Warning".
+ res.setHeader("Warning", `299 - ${quoteWarningText(notice.message)}`);
+
+ const originalJson = res.json.bind(res);
+ res.json = (body: any) => {
+ if (body && typeof body === "object" && !Array.isArray(body)) {
+ const existing = Array.isArray(body.warnings) ? body.warnings : [];
+ const annotated: any = {
+ ...body,
+ warnings: [...existing, notice.message],
+ };
+ if (notice.replacement && annotated.replacement === undefined) {
+ annotated.replacement = notice.replacement;
+ }
+ return originalJson(annotated);
+ }
+ return originalJson(body);
+ };
+}
+
export function deprecationMiddleware(key: DeprecationKey) {
const dep: Deprecation = DEPRECATIONS[key];
return (req: Request, res: Response, next: NextFunction) => {
@@ -112,27 +141,11 @@ export function deprecationMiddleware(key: DeprecationKey) {
if (dep.replacement) {
links.push(`<${dep.replacement}>; rel="successor-version"`);
}
- if (links.length > 0) res.setHeader("Link", links.join(", "));
-
- // RFC 7234 Warning header, code 299 = "Miscellaneous Persistent Warning".
- res.setHeader("Warning", `299 - ${quoteWarningText(dep.message)}`);
-
- const originalJson = res.json.bind(res);
- res.json = (body: any) => {
- if (body && typeof body === "object" && !Array.isArray(body)) {
- // Copy: the research controllers log the same object after responding.
- const existing = Array.isArray(body.warnings) ? body.warnings : [];
- const annotated: any = {
- ...body,
- warnings: [...existing, dep.message],
- };
- if (dep.replacement && annotated.replacement === undefined) {
- annotated.replacement = dep.replacement;
- }
- return originalJson(annotated);
- }
- return originalJson(body);
- };
+ applyNotice(res, {
+ message: dep.message,
+ links,
+ replacement: dep.replacement,
+ });
next();
};
}
diff --git a/apps/api/src/lib/search-query-builder.ts b/apps/api/src/lib/search-query-builder.ts
index 0102db692..bcbd6d5aa 100644
--- a/apps/api/src/lib/search-query-builder.ts
+++ b/apps/api/src/lib/search-query-builder.ts
@@ -38,6 +38,16 @@ const DEFAULT_RESEARCH_SITES = [
"medrxiv.org",
];
+export function hasCategory(
+ categories: unknown,
+ type: CategoryInput["type"],
+): boolean {
+ return (
+ Array.isArray(categories) &&
+ categories.some(c => (typeof c === "string" ? c : c?.type) === type)
+ );
+}
+
/**
* Builds a search query with category filters
* @param baseQuery The base search query
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index c6f9f1646..157c798ca 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -5,7 +5,10 @@ import { RateLimiterMode } from "../types";
import { registerMcpActionLogReadRoute } from "./mcp-action-logs";
import { SEARCH_CREDITS_FEATURE_ID } from "../services/autumn/autumn.service";
import expressWs from "express-ws";
-import { searchController } from "../controllers/v2/search";
+import {
+ researchCategoryNoticeMiddleware,
+ searchController,
+} from "../controllers/v2/search";
import { feedbackController } from "../controllers/v2/feedback/controller";
import { searchFeedbackController } from "../controllers/v2/search-feedback";
import { scrapeController } from "../controllers/v2/scrape";
@@ -185,6 +188,7 @@ registerMcpActionLogReadRoute(
v2Router.post(
"/search",
+ researchCategoryNoticeMiddleware,
authMiddleware(RateLimiterMode.Search, { allowKeyless: true }),
countryCheck,
checkCreditsMiddleware(undefined, SEARCH_CREDITS_FEATURE_ID),
diff --git a/apps/api/src/search/developer.ts b/apps/api/src/search/developer.ts
index 78e5c6e51..2304ccd88 100644
--- a/apps/api/src/search/developer.ts
+++ b/apps/api/src/search/developer.ts
@@ -1,16 +1,12 @@
import type { Logger } from "winston";
import type { WebSearchResult } from "../lib/entities";
import { fetchResearchUpstream } from "../lib/research-upstream";
-import type { CategoryOption } from "../lib/search-query-builder";
+import { hasCategory, type CategoryOption } from "../lib/search-query-builder";
const DEVELOPER_QUERY_KEYS = ["query", "k"];
export function wantsDeveloperCategory(categories?: CategoryOption[]): boolean {
- return (categories ?? []).some(category =>
- typeof category === "string"
- ? category === "developer"
- : category.type === "developer",
- );
+ return hasCategory(categories, "developer");
}
export async function searchDeveloperCategory(