Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): sunset deep-research and llmstxt, with surface keys and migration links (#5606)
Gergő Móricz603d3ec4a7f7
diff --git a/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.ts b/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.tsindex 314a4febf..e20c81ed9 100644--- a/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.ts+++ b/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.ts@@ -2,9 +2,9 @@ import express from "express";import request from "supertest";import { deprecationMiddleware } from "../../lib/deprecations";-// Every pre-existing deprecation entry and its replacement. Pinned so the-// github entry cannot change anyone else's wire output. All 13 shipped in-// #3469 on 2026-05-06, which is the date they now emit.+// Pre-existing deprecation entries and their replacements. Pinned so the+// github entry cannot change anyone else's wire output. All shipped in #3469+// on 2026-05-06, which is the date they now emit.const LEGACY_DEPRECATED_AT = "@1778025600";const EXISTING = {@@ -12,10 +12,6 @@ const EXISTING = {v1_extract_status: "/v2/scrape",v2_extract: "/v2/scrape",v2_extract_status: "/v2/scrape",- v1_deep_research: "/v2/search",- v1_deep_research_status: "/v2/search",- v1_llmstxt: undefined,- v1_llmstxt_status: undefined,v0_scrape: "/v2/scrape",v0_crawl: "/v2/crawl",v0_crawl_status: "/v2/crawl/:jobId",@@ -115,3 +111,43 @@ describe("pre-existing deprecated routes are unchanged", () => {expect(str.text).toBe('"hello"');});});++describe("deep-research and llmstxt entries", () => {+ const CASES = {+ v1_deep_research: {+ replacement: "/v2/agent",+ docs: "https://docs.firecrawl.dev/migrating-from-deep-research",+ },+ v1_deep_research_status: {+ replacement: "/v2/agent/:jobId",+ docs: "https://docs.firecrawl.dev/migrating-from-deep-research",+ },+ v1_llmstxt: {+ replacement: undefined,+ docs: "https://docs.firecrawl.dev/migrating-from-llmstxt",+ },+ v1_llmstxt_status: {+ replacement: undefined,+ docs: "https://docs.firecrawl.dev/migrating-from-llmstxt",+ },+ } as const;++ for (const key of Object.keys(CASES) as (keyof typeof CASES)[]) {+ const { replacement, docs } = CASES[key];++ it(`${key} links its migration guide and replacement`, async () => {+ const app = express();+ app.get("/probe", deprecationMiddleware(key), (_req, res) => {+ res.status(200).json({ success: true });+ });+ const res = await request(app).get("/probe");++ expect(res.headers["deprecation"]).toBe(LEGACY_DEPRECATED_AT);+ expect(res.headers["sunset"]).toBe("Thu, 10 Dec 2026 15:00:00 GMT");+ const links = [`<${docs}>; rel="deprecation"`];+ if (replacement) links.push(`<${replacement}>; rel="successor-version"`);+ expect(res.headers["link"]).toBe(links.join(", "));+ expect(res.body.replacement).toBe(replacement);+ });+ }+});diff --git a/apps/api/src/__tests__/snips/v1/deprecation.test.ts b/apps/api/src/__tests__/snips/v1/deprecation.test.tsindex dca867ad3..832e0cc81 100644--- a/apps/api/src/__tests__/snips/v1/deprecation.test.ts+++ b/apps/api/src/__tests__/snips/v1/deprecation.test.ts@@ -34,7 +34,9 @@ describe("Deprecation warnings on legacy endpoints", () => {true,);expect(res.body.replacement).toBeUndefined();- expect(res.headers["link"]).toBeUndefined();+ expect(res.headers["link"]).toBe(+ '<https://docs.firecrawl.dev/migrating-from-llmstxt>; rel="deprecation"',+ );}, 30000);it("GET /v1/llmstxt/:jobId still emits warnings on 404", async () => {@@ -66,14 +68,14 @@ describe("Deprecation warnings on legacy endpoints", () => {expect(res.statusCode).toBe(200);expect(res.headers["deprecation"]).toBe("@1778025600");expect(res.headers["warning"]).toMatch(/deep-research/i);- expect(res.headers["link"]).toContain(- '</v2/search>; rel="successor-version"',+ expect(res.headers["link"]).toBe(+ '<https://docs.firecrawl.dev/migrating-from-deep-research>; rel="deprecation", </v2/agent>; rel="successor-version"',);expect(Array.isArray(res.body.warnings)).toBe(true);expect(res.body.warnings.some((w: string) => /deep-research/i.test(w)),).toBe(true);- expect(res.body.replacement).toBe("/v2/search");+ expect(res.body.replacement).toBe("/v2/agent");}, 30000);it("POST /v1/scrape points at /v2/scrape and the migration guide", async () => {diff --git a/apps/api/src/lib/deprecations.ts b/apps/api/src/lib/deprecations.tsindex 4b04e1fb0..faa62a39d 100644--- a/apps/api/src/lib/deprecations.ts+++ b/apps/api/src/lib/deprecations.ts@@ -68,22 +68,38 @@ const DEPRECATIONS = {sunset: "Tue, 03 Nov 2026 23:59:59 GMT",},v1_deep_research: {- message: "/v1/deep-research is deprecated. Use /v2/search instead.",- replacement: "/v2/search",+ message:+ "/v1/deep-research is deprecated. Use /v2/agent instead, or /v2/search with scrapeOptions to run your own research loop.",+ replacement: "/v2/agent",+ docs: "https://docs.firecrawl.dev/migrating-from-deep-research",deprecatedAt: "@1778025600",+ sunset: "Thu, 10 Dec 2026 15:00:00 GMT",+ surface: "deep_research",},v1_deep_research_status: {- message: "/v1/deep-research/:jobId is deprecated. Use /v2/search instead.",- replacement: "/v2/search",+ message:+ "/v1/deep-research/:jobId is deprecated. Use /v2/agent instead, and read results from /v2/agent/:jobId.",+ replacement: "/v2/agent/:jobId",+ docs: "https://docs.firecrawl.dev/migrating-from-deep-research",deprecatedAt: "@1778025600",+ sunset: "Thu, 10 Dec 2026 15:00:00 GMT",+ surface: "deep_research",},v1_llmstxt: {- message: "/v1/llmstxt is deprecated and will not be replaced.",+ message:+ "/v1/llmstxt is deprecated. Build llms.txt from /v2/map and /v2/batch/scrape instead.",+ docs: "https://docs.firecrawl.dev/migrating-from-llmstxt",deprecatedAt: "@1778025600",+ sunset: "Thu, 10 Dec 2026 15:00:00 GMT",+ surface: "llmstxt",},v1_llmstxt_status: {- message: "/v1/llmstxt/:jobId is deprecated and will not be replaced.",+ message:+ "/v1/llmstxt/:jobId is deprecated. Build llms.txt from /v2/map and /v2/batch/scrape instead.",+ docs: "https://docs.firecrawl.dev/migrating-from-llmstxt",deprecatedAt: "@1778025600",+ sunset: "Thu, 10 Dec 2026 15:00:00 GMT",+ surface: "llmstxt",},v0_scrape: {message: