Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): deprecation warnings on legacy endpoints (#3469)
Abimael Martell4d5c1dc9636a
diff --git a/apps/api/src/__tests__/snips/v1/deprecation.test.ts b/apps/api/src/__tests__/snips/v1/deprecation.test.tsnew file mode 100644index 000000000..e58556781--- /dev/null+++ b/apps/api/src/__tests__/snips/v1/deprecation.test.ts@@ -0,0 +1,93 @@+import { describe, it, expect, beforeAll } from "@jest/globals";+import request from "supertest";+import { TEST_API_URL } from "../lib";+import { idmux, Identity } from "./lib";++let identity: Identity;++beforeAll(async () => {+ identity = await idmux({+ name: "deprecation",+ concurrency: 10,+ credits: 1000,+ });+}, 10000);++describe("Deprecation warnings on legacy endpoints", () => {+ it("POST /v1/llmstxt enqueues with Deprecation header and warnings in body", async () => {+ const res = await request(TEST_API_URL)+ .post("/v1/llmstxt")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({ url: "https://firecrawl.dev" });++ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ expect(res.headers["deprecation"]).toBe("true");+ expect(res.headers["warning"]).toMatch(/^299 - "/);+ expect(res.headers["warning"]).toMatch(/llmstxt/i);+ expect(Array.isArray(res.body.warnings)).toBe(true);+ expect(res.body.warnings.some((w: string) => /llmstxt/i.test(w))).toBe(+ true,+ );+ expect(res.body.warnings.some((w: string) => /deprecated/i.test(w))).toBe(+ true,+ );+ expect(res.body.replacement).toBeUndefined();+ expect(res.headers["link"]).toBeUndefined();+ }, 30000);++ it("GET /v1/llmstxt/:jobId still emits warnings on 404", async () => {+ const res = await request(TEST_API_URL)+ .get(`/v1/llmstxt/${crypto.randomUUID()}`)+ .set("Authorization", `Bearer ${identity.apiKey}`);++ expect(res.statusCode).toBe(404);+ expect(res.headers["deprecation"]).toBe("true");+ expect(res.headers["warning"]).toMatch(/deprecated/i);+ expect(Array.isArray(res.body.warnings)).toBe(true);+ expect(res.body.warnings.some((w: string) => /deprecated/i.test(w))).toBe(+ true,+ );+ }, 30000);++ it("POST /v1/deep-research returns warnings and successor-version Link", async () => {+ const res = await request(TEST_API_URL)+ .post("/v1/deep-research")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({+ query: "what is firecrawl",+ maxDepth: 1,+ maxUrls: 1,+ timeLimit: 60,+ });++ expect(res.statusCode).toBe(200);+ expect(res.headers["deprecation"]).toBe("true");+ expect(res.headers["warning"]).toMatch(/deep-research/i);+ expect(res.headers["link"]).toContain(+ '</v2/search>; rel="successor-version"',+ );+ expect(Array.isArray(res.body.warnings)).toBe(true);+ expect(+ res.body.warnings.some((w: string) => /deep-research/i.test(w)),+ ).toBe(true);+ expect(res.body.replacement).toBe("/v2/search");+ }, 30000);++ it("non-deprecated endpoints do not emit Deprecation header or warnings", async () => {+ const res = await request(TEST_API_URL)+ .post("/v1/scrape")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({ url: "https://firecrawl.dev" });++ expect(res.headers["deprecation"]).toBeUndefined();+ expect(res.headers["warning"]).toBeUndefined();+ if (res.body && typeof res.body === "object") {+ expect(res.body.warnings).toBeUndefined();+ expect(res.body.replacement).toBeUndefined();+ }+ }, 60000);+});diff --git a/apps/api/src/lib/deprecations.ts b/apps/api/src/lib/deprecations.tsnew file mode 100644index 000000000..02ede0a01--- /dev/null+++ b/apps/api/src/lib/deprecations.ts@@ -0,0 +1,109 @@+import { NextFunction, Request, Response } from "express";++interface Deprecation {+ message: string;+ replacement?: string;+ sunset?: string;+ docs?: string;+}++const DEPRECATIONS = {+ v1_extract: {+ message:+ "/v1/extract is deprecated. Use /v2/scrape with formats including a 'json' format object.",+ replacement: "/v2/scrape",+ },+ v1_extract_status: {+ message:+ "/v1/extract/:jobId is deprecated. Use /v2/scrape with formats including a 'json' format object.",+ replacement: "/v2/scrape",+ },+ v2_extract: {+ message:+ "/v2/extract is deprecated. Use /v2/scrape with formats including a 'json' format object.",+ replacement: "/v2/scrape",+ },+ v2_extract_status: {+ message:+ "/v2/extract/:jobId is deprecated. Use /v2/scrape with formats including a 'json' format object.",+ replacement: "/v2/scrape",+ },+ v1_deep_research: {+ message: "/v1/deep-research is deprecated. Use /v2/search instead.",+ replacement: "/v2/search",+ },+ v1_deep_research_status: {+ message: "/v1/deep-research/:jobId is deprecated. Use /v2/search instead.",+ replacement: "/v2/search",+ },+ v1_llmstxt: {+ message: "/v1/llmstxt is deprecated and will not be replaced.",+ },+ v1_llmstxt_status: {+ message: "/v1/llmstxt/:jobId is deprecated and will not be replaced.",+ },+ v0_scrape: {+ message: "/v0/scrape is deprecated. Use /v2/scrape instead.",+ replacement: "/v2/scrape",+ },+ v0_crawl: {+ message: "/v0/crawl is deprecated. Use /v2/crawl instead.",+ replacement: "/v2/crawl",+ },+ v0_crawl_status: {+ message:+ "/v0/crawl/status/:jobId is deprecated. Use /v2/crawl/:jobId instead.",+ replacement: "/v2/crawl/:jobId",+ },+ v0_crawl_cancel: {+ message:+ "/v0/crawl/cancel/:jobId is deprecated. Use DELETE /v2/crawl/:jobId instead.",+ replacement: "/v2/crawl/:jobId",+ },+ v0_search: {+ message: "/v0/search is deprecated. Use /v2/search instead.",+ replacement: "/v2/search",+ },+} as const satisfies Record<string, Deprecation>;++type DeprecationKey = keyof typeof DEPRECATIONS;++// RFC 7234 quoted-string: escape backslash and double quote.+function quoteWarningText(s: string): string {+ return `"${s.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"`;+}++export function deprecationMiddleware(key: DeprecationKey) {+ const dep: Deprecation = DEPRECATIONS[key];+ return (req: Request, res: Response, next: NextFunction) => {+ // RFC 9745 Deprecation header.+ res.setHeader("Deprecation", "true");+ // RFC 8594 Sunset header.+ if (dep.sunset) res.setHeader("Sunset", dep.sunset);++ // RFC 8288 Link relations: "deprecation" (RFC 9745) for docs, and+ // "successor-version" (RFC 5829) for the replacement endpoint.+ const links: string[] = [];+ if (dep.docs) links.push(`<${dep.docs}>; rel="deprecation"`);+ if (dep.replacement) {+ links.push(`<${dep.replacement}>; rel="successor-version"`);+ }+ if (links.length > 0) res.setHeader("Link", links.join(", "));++ // RFC 7234 Warning header, code 299 = "Miscellaneous Persistent Warning".+ res.setHeader("Warning", `299 - ${quoteWarningText(dep.message)}`);++ const originalJson = res.json.bind(res);+ res.json = (body: any) => {+ if (body && typeof body === "object" && !Array.isArray(body)) {+ const existing = Array.isArray(body.warnings) ? body.warnings : [];+ body.warnings = [...existing, dep.message];+ if (dep.replacement && body.replacement === undefined) {+ body.replacement = dep.replacement;+ }+ }+ return originalJson(body);+ };+ next();+ };+}diff --git a/apps/api/src/routes/v0.ts b/apps/api/src/routes/v0.tsindex 772cab581..550a7b8b6 100644--- a/apps/api/src/routes/v0.ts+++ b/apps/api/src/routes/v0.ts@@ -7,19 +7,36 @@ import { crawlCancelController } from "../../src/controllers/v0/crawl-cancel";import { keyAuthController } from "../../src/controllers/v0/keyAuth";import { livenessController } from "../controllers/v0/liveness";import { readinessController } from "../controllers/v0/readiness";+import { deprecationMiddleware } from "../lib/deprecations";export const v0Router = express.Router();-v0Router.post("/v0/scrape", scrapeController);-v0Router.post("/v0/crawl", crawlController);-v0Router.get("/v0/crawl/status/:jobId", crawlStatusController);-v0Router.delete("/v0/crawl/cancel/:jobId", crawlCancelController);+v0Router.post(+ "/v0/scrape",+ deprecationMiddleware("v0_scrape"),+ scrapeController,+);+v0Router.post("/v0/crawl", deprecationMiddleware("v0_crawl"), crawlController);+v0Router.get(+ "/v0/crawl/status/:jobId",+ deprecationMiddleware("v0_crawl_status"),+ crawlStatusController,+);+v0Router.delete(+ "/v0/crawl/cancel/:jobId",+ deprecationMiddleware("v0_crawl_cancel"),+ crawlCancelController,+);// Auth route for key based authenticationv0Router.get("/v0/keyAuth", keyAuthController);// Search routes-v0Router.post("/v0/search", searchController);+v0Router.post(+ "/v0/search",+ deprecationMiddleware("v0_search"),+ searchController,+);// Health/Probe routesv0Router.get("/v0/health/liveness", livenessController);diff --git a/apps/api/src/routes/v1.ts b/apps/api/src/routes/v1.tsindex 5b8f0b67d..c698eb9b0 100644--- a/apps/api/src/routes/v1.ts+++ b/apps/api/src/routes/v1.ts@@ -46,6 +46,7 @@ import {createX402RouteConfig,isX402Enabled,} from "../lib/x402";+import { deprecationMiddleware } from "../lib/deprecations";expressWs(express());@@ -222,6 +223,7 @@ v1Router.ws("/crawl/:jobId", crawlStatusWSController);v1Router.post("/extract",authMiddleware(RateLimiterMode.Extract),+ deprecationMiddleware("v1_extract"),countryCheck,checkCreditsMiddleware(20),wrap(extractController),@@ -230,12 +232,14 @@ v1Router.post(v1Router.get("/extract/:jobId",authMiddleware(RateLimiterMode.ExtractStatus),+ deprecationMiddleware("v1_extract_status"),wrap(extractStatusController),);v1Router.post("/llmstxt",authMiddleware(RateLimiterMode.Scrape),+ deprecationMiddleware("v1_llmstxt"),countryCheck,blocklistMiddleware,wrap(generateLLMsTextController),@@ -244,12 +248,14 @@ v1Router.post(v1Router.get("/llmstxt/:jobId",authMiddleware(RateLimiterMode.CrawlStatus),+ deprecationMiddleware("v1_llmstxt_status"),wrap(generateLLMsTextStatusController),);v1Router.post("/deep-research",authMiddleware(RateLimiterMode.Crawl),+ deprecationMiddleware("v1_deep_research"),countryCheck,checkCreditsMiddleware(1),wrap(deepResearchController),@@ -258,6 +264,7 @@ v1Router.post(v1Router.get("/deep-research/:jobId",authMiddleware(RateLimiterMode.CrawlStatus),+ deprecationMiddleware("v1_deep_research_status"),wrap(deepResearchStatusController),);diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 5ebf901e2..0c738f92c 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -45,6 +45,7 @@ import {createX402RouteConfig,isX402Enabled,} from "../lib/x402";+import { deprecationMiddleware } from "../lib/deprecations";import { agentController } from "../controllers/v2/agent";import { agentStatusController } from "../controllers/v2/agent-status";import { agentCancelController } from "../controllers/v2/agent-cancel";@@ -381,6 +382,7 @@ v2Router.get(v2Router.post("/extract",authMiddleware(RateLimiterMode.Extract),+ deprecationMiddleware("v2_extract"),countryCheck,checkCreditsMiddleware(20),blocklistMiddleware,@@ -390,6 +392,7 @@ v2Router.post(v2Router.get("/extract/:jobId",authMiddleware(RateLimiterMode.ExtractStatus),+ deprecationMiddleware("v2_extract_status"),validateJobIdParam,wrap(extractStatusController),);