Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
test(api): stop deep research and llms.txt snips at their sunset and check the 410 (#5687)
Gergő Móricz4c4275fa8a2c
diff --git a/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.ts b/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.tsindex 596fb1b2a..e5b62e5e4 100644--- a/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.ts+++ b/apps/api/src/__tests__/routes/deprecations-existing-routes.routes.test.ts@@ -1,6 +1,6 @@import express from "express";import request from "supertest";-import { deprecationMiddleware } from "../../lib/deprecations";+import { deprecationMiddleware, sunsetOf } from "../../lib/deprecations";// Pre-existing deprecation entries and their replacements. Pinned so the// github entry cannot change anyone else's wire output. All shipped in #3469@@ -140,10 +140,17 @@ describe("deep-research and llmstxt entries", () => {},} as const;+ afterEach(() => {+ vi.useRealTimers();+ });+for (const key of Object.keys(CASES) as (keyof typeof CASES)[]) {const { replacement, docs } = CASES[key];it(`${key} links its migration guide and replacement`, async () => {+ // Just before the entry's sunset, so the route still serves.+ vi.useFakeTimers({ toFake: ["Date"] });+ vi.setSystemTime(sunsetOf(key) - 1);const app = express();app.get("/probe", deprecationMiddleware(key), (_req, res) => {res.status(200).json({ success: true });@@ -151,7 +158,7 @@ describe("deep-research and llmstxt entries", () => {const res = await request(app).get("/probe");expect(res.headers["deprecation"]).toBe(LEGACY_DEPRECATED_AT);- expect(res.headers["sunset"]).toBe("Thu, 10 Dec 2026 15:00:00 GMT");+ expect(res.headers["sunset"]).toBe(new Date(sunsetOf(key)).toUTCString());const links = [`<${docs}>; rel="deprecation"`];if (replacement) links.push(`<${replacement}>; rel="successor-version"`);expect(res.headers["link"]).toBe(links.join(", "));diff --git a/apps/api/src/__tests__/snips/v1/deep-research.test.ts b/apps/api/src/__tests__/snips/v1/deep-research.test.tsindex 4e606b8cd..1df8a91c0 100644--- a/apps/api/src/__tests__/snips/v1/deep-research.test.ts+++ b/apps/api/src/__tests__/snips/v1/deep-research.test.ts@@ -1,5 +1,5 @@import { describe, it, expect, beforeAll } from "vitest";-import { deepResearch, idmux, Identity } from "./lib";+import { DEEP_RESEARCH_LIVE, deepResearch, idmux, Identity } from "./lib";import { describeIf, HAS_AI, TEST_PRODUCTION } from "../lib";let identity: Identity;@@ -12,7 +12,11 @@ beforeAll(async () => {});}, 10000);-describeIf(TEST_PRODUCTION || HAS_AI)("Deep Research", () => {+const describeDeepResearch = describeIf(+ (TEST_PRODUCTION || HAS_AI) && DEEP_RESEARCH_LIVE,+);++describeDeepResearch("Deep Research", () => {it("successfully completes a basic research query", async () => {const query = "What is artificial intelligence?";diff --git a/apps/api/src/__tests__/snips/v1/deprecation.test.ts b/apps/api/src/__tests__/snips/v1/deprecation.test.tsindex d546ed857..e3728f5dd 100644--- a/apps/api/src/__tests__/snips/v1/deprecation.test.ts+++ b/apps/api/src/__tests__/snips/v1/deprecation.test.ts@@ -1,9 +1,18 @@import { describe, it, expect, beforeAll } from "vitest";import request from "supertest";import { itIf, TEST_API_URL } from "../lib";-import { idmux, Identity, V1_LIVE } from "./lib";+import { sunsetOf } from "../../../lib/deprecations";+import {+ DEEP_RESEARCH_LIVE,+ idmux,+ Identity,+ LLMSTXT_LIVE,+ V1_LIVE,+} from "./lib";const itV1 = itIf(V1_LIVE);+const itLlmsTxt = itIf(LLMSTXT_LIVE);+const itDeepResearch = itIf(DEEP_RESEARCH_LIVE);let identity: Identity;@@ -15,70 +24,100 @@ beforeAll(async () => {});}, 10000);+async function expectRemoved(+ key: Parameters<typeof sunsetOf>[0],+ method: "get" | "post",+ path: string,+ replacement: string | undefined,+ docs: string,+) {+ const res = await request(TEST_API_URL)+ [method](path)+ .set("Authorization", `Bearer ${identity.apiKey}`);++ expect(res.statusCode).toBe(410);+ expect(res.body.code).toBe("ENDPOINT_REMOVED");+ expect(res.body.replacement).toBe(replacement);+ expect(res.body.documentation_url).toBe(docs);+ expect(res.headers["sunset"]).toBe(new Date(sunsetOf(key)).toUTCString());+}+describe("Deprecation warnings on legacy endpoints", () => {- it("POST /v1/llmstxt enqueues with Deprecation header and warnings in body", async () => {- const res = await request(TEST_API_URL)- .post("/v1/llmstxt")- .set("Authorization", `Bearer ${identity.apiKey}`)- .set("Content-Type", "application/json")- .send({ url: "https://firecrawl.dev" });-- expect(res.statusCode).toBe(200);- expect(res.body.success).toBe(true);- expect(res.headers["deprecation"]).toBe("@1778025600");- expect(res.headers["warning"]).toMatch(/^299 - "/);- expect(res.headers["warning"]).toMatch(/llmstxt/i);- expect(Array.isArray(res.body.warnings)).toBe(true);- expect(res.body.warnings.some((w: string) => /llmstxt/i.test(w))).toBe(- true,- );- expect(res.body.warnings.some((w: string) => /deprecated/i.test(w))).toBe(- true,- );- expect(res.body.replacement).toBeUndefined();- expect(res.headers["link"]).toBe(- '<https://docs.firecrawl.dev/migrating-from-llmstxt>; rel="deprecation"',- );- }, 30000);-- it("GET /v1/llmstxt/:jobId still emits warnings on 404", async () => {- const res = await request(TEST_API_URL)- .get(`/v1/llmstxt/${crypto.randomUUID()}`)- .set("Authorization", `Bearer ${identity.apiKey}`);+ itLlmsTxt(+ "POST /v1/llmstxt enqueues with Deprecation header and warnings in body",+ async () => {+ const res = await request(TEST_API_URL)+ .post("/v1/llmstxt")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({ url: "https://firecrawl.dev" });- expect(res.statusCode).toBe(404);- expect(res.headers["deprecation"]).toBe("@1778025600");- expect(res.headers["warning"]).toMatch(/deprecated/i);- expect(Array.isArray(res.body.warnings)).toBe(true);- expect(res.body.warnings.some((w: string) => /deprecated/i.test(w))).toBe(- true,- );- }, 30000);+ expect(res.statusCode).toBe(200);+ expect(res.body.success).toBe(true);+ expect(res.headers["deprecation"]).toBe("@1778025600");+ expect(res.headers["warning"]).toMatch(/^299 - "/);+ expect(res.headers["warning"]).toMatch(/llmstxt/i);+ expect(Array.isArray(res.body.warnings)).toBe(true);+ expect(res.body.warnings.some((w: string) => /llmstxt/i.test(w))).toBe(+ true,+ );+ expect(res.body.warnings.some((w: string) => /deprecated/i.test(w))).toBe(+ true,+ );+ expect(res.body.replacement).toBeUndefined();+ expect(res.headers["link"]).toBe(+ '<https://docs.firecrawl.dev/migrating-from-llmstxt>; rel="deprecation"',+ );+ },+ 30000,+ );- it("POST /v1/deep-research returns warnings and successor-version Link", async () => {- const res = await request(TEST_API_URL)- .post("/v1/deep-research")- .set("Authorization", `Bearer ${identity.apiKey}`)- .set("Content-Type", "application/json")- .send({- query: "what is firecrawl",- maxDepth: 1,- maxUrls: 1,- timeLimit: 60,- });-- expect(res.statusCode).toBe(200);- expect(res.headers["deprecation"]).toBe("@1778025600");- expect(res.headers["warning"]).toMatch(/deep-research/i);- expect(res.headers["link"]).toBe(- '<https://docs.firecrawl.dev/migrating-from-deep-research>; rel="deprecation", </v2/agent>; rel="successor-version"',- );- expect(Array.isArray(res.body.warnings)).toBe(true);- expect(- res.body.warnings.some((w: string) => /deep-research/i.test(w)),- ).toBe(true);- expect(res.body.replacement).toBe("/v2/agent");- }, 30000);+ itLlmsTxt(+ "GET /v1/llmstxt/:jobId still emits warnings on 404",+ async () => {+ const res = await request(TEST_API_URL)+ .get(`/v1/llmstxt/${crypto.randomUUID()}`)+ .set("Authorization", `Bearer ${identity.apiKey}`);++ expect(res.statusCode).toBe(404);+ expect(res.headers["deprecation"]).toBe("@1778025600");+ expect(res.headers["warning"]).toMatch(/deprecated/i);+ expect(Array.isArray(res.body.warnings)).toBe(true);+ expect(res.body.warnings.some((w: string) => /deprecated/i.test(w))).toBe(+ true,+ );+ },+ 30000,+ );++ itDeepResearch(+ "POST /v1/deep-research returns warnings and successor-version Link",+ async () => {+ const res = await request(TEST_API_URL)+ .post("/v1/deep-research")+ .set("Authorization", `Bearer ${identity.apiKey}`)+ .set("Content-Type", "application/json")+ .send({+ query: "what is firecrawl",+ maxDepth: 1,+ maxUrls: 1,+ timeLimit: 60,+ });++ expect(res.statusCode).toBe(200);+ expect(res.headers["deprecation"]).toBe("@1778025600");+ expect(res.headers["warning"]).toMatch(/deep-research/i);+ expect(res.headers["link"]).toBe(+ '<https://docs.firecrawl.dev/migrating-from-deep-research>; rel="deprecation", </v2/agent>; rel="successor-version"',+ );+ expect(Array.isArray(res.body.warnings)).toBe(true);+ expect(+ res.body.warnings.some((w: string) => /deep-research/i.test(w)),+ ).toBe(true);+ expect(res.body.replacement).toBe("/v2/agent");+ },+ 30000,+ );itV1("POST /v1/scrape points at /v2/scrape and the migration guide",@@ -160,6 +199,45 @@ describe("Deprecation warnings on legacy endpoints", () => {30000,);+ itIf(!DEEP_RESEARCH_LIVE).each([+ ["v1_deep_research", "post", "/v1/deep-research", "/v2/agent"],+ [+ "v1_deep_research_status",+ "get",+ `/v1/deep-research/${crypto.randomUUID()}`,+ "/v2/agent/:jobId",+ ],+ ] as const)(+ "%s: %s %s answers 410 after the deep research sunset",+ async (key, method, path, replacement) => {+ await expectRemoved(+ key,+ method,+ path,+ replacement,+ "https://docs.firecrawl.dev/migrating-from-deep-research",+ );+ },+ 30000,+ );++ itIf(!LLMSTXT_LIVE).each([+ ["v1_llmstxt", "post", "/v1/llmstxt"],+ ["v1_llmstxt_status", "get", `/v1/llmstxt/${crypto.randomUUID()}`],+ ] as const)(+ "%s: %s %s answers 410 after the llms.txt sunset",+ async (key, method, path) => {+ await expectRemoved(+ key,+ method,+ path,+ undefined,+ "https://docs.firecrawl.dev/migrating-from-llmstxt",+ );+ },+ 30000,+ );+it("non-deprecated endpoints do not emit Deprecation header or warnings", async () => {const res = await request(TEST_API_URL).get("/v2/team/credit-usage")diff --git a/apps/api/src/__tests__/snips/v1/lib.ts b/apps/api/src/__tests__/snips/v1/lib.tsindex 567e622bc..affc0d965 100644--- a/apps/api/src/__tests__/snips/v1/lib.ts+++ b/apps/api/src/__tests__/snips/v1/lib.ts@@ -33,6 +33,10 @@ export { scrapeTimeout, indexCooldown, Identity, idmux };export const V1_LIVE = Date.now() < sunsetOf("v1_scrape");export const describeV1 = (cond = true) => describeIf(V1_LIVE && cond);+// Deep research and llms.txt answer 410 from their own sunsets.+export const DEEP_RESEARCH_LIVE = Date.now() < sunsetOf("v1_deep_research");+export const LLMSTXT_LIVE = Date.now() < sunsetOf("v1_llmstxt");+const pollSleep = async () => new Promise(r => setTimeout(r, 50));export async function scrapeRaw(body: ScrapeRequestInput, identity: Identity) {