Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): sunset dates and per-team extensions for deprecated routes (#5605)
Gergő Móriczbf6739d95cac
diff --git a/apps/api/src/__tests__/routes/deprecations-sunset.routes.test.ts b/apps/api/src/__tests__/routes/deprecations-sunset.routes.test.tsnew file mode 100644index 000000000..67e325e13--- /dev/null+++ b/apps/api/src/__tests__/routes/deprecations-sunset.routes.test.ts@@ -0,0 +1,230 @@+import express from "express";+import request from "supertest";++const mocks = vi.hoisted(() => ({ getApiKeyTeamFlags: vi.fn() }));++vi.mock("../../controllers/auth", () => ({+ getApiKeyTeamFlags: mocks.getApiKeyTeamFlags,+}));++import {+ deprecationMiddleware,+ hasSunsetExtension,+ sunsetInstant,+} from "../../lib/deprecations";++// The only entry with a sunset today: Tue, 03 Nov 2026 23:59:59 GMT.+const SUNSET_KEY = "v2_research_github_search";+const BEFORE_SUNSET = new Date("2026-11-01T00:00:00Z");+const AFTER_SUNSET = new Date("2026-11-10T00:00:00Z");++function appFor(+ key: Parameters<typeof deprecationMiddleware>[0],+ acuc?: { flags: unknown },+) {+ const app = express();+ app.get(+ "/probe",+ (req: any, _res, next) => {+ if (acuc) req.acuc = acuc;+ next();+ },+ deprecationMiddleware(key),+ (_req, res) => {+ res.status(200).json({ success: true });+ },+ );+ return app;+}++function extendedUntil(until: string, surface = SUNSET_KEY) {+ return { deprecationExtensions: { [surface]: until } };+}++beforeEach(() => {+ vi.useFakeTimers({ toFake: ["Date"] });+ mocks.getApiKeyTeamFlags.mockReset();+});++afterEach(() => {+ vi.useRealTimers();+});++describe("sunsetInstant", () => {+ it("reads an exact IMF-fixdate", () => {+ expect(sunsetInstant("k", "Tue, 03 Nov 2026 23:59:59 GMT")).toBe(+ Date.parse("2026-11-03T23:59:59Z"),+ );+ });++ it("is never reached without a sunset", () => {+ expect(sunsetInstant("k", undefined)).toBe(Infinity);+ });++ it.each([+ "Tue, 31 Nov 2026 23:59:59 GMT",+ "Mon, 03 Nov 2026 23:59:59 GMT",+ "3 Nov 2026",+ "2026-11-03",+ "soon",+ ])("refuses %s", sunset => {+ expect(() => sunsetInstant("k", sunset)).toThrow(/unreadable sunset/);+ });+});++describe("hasSunsetExtension", () => {+ const now = Date.parse("2026-12-01T00:00:00Z");++ it("reads the flag under the entry's surface", () => {+ expect(+ hasSunsetExtension(+ "v0_scrape",+ { surface: "v0" },+ extendedUntil("2027-01-31", "v0"),+ now,+ ),+ ).toBe(true);+ });++ it("ignores a flag under the entry key when the entry has a surface", () => {+ expect(+ hasSunsetExtension(+ "v0_scrape",+ { surface: "v0" },+ extendedUntil("2027-01-31", "v0_scrape"),+ now,+ ),+ ).toBe(false);+ });++ it("falls back to the entry key without a surface", () => {+ expect(+ hasSunsetExtension(+ "v1_llmstxt",+ {},+ extendedUntil("2027-01-31", "v1_llmstxt"),+ now,+ ),+ ).toBe(true);+ });++ it("ends at the flag's instant", () => {+ const flags = extendedUntil("2026-12-01T00:00:00Z", "v0");+ expect(+ hasSunsetExtension("v0_scrape", { surface: "v0" }, flags, now - 1),+ ).toBe(true);+ expect(hasSunsetExtension("v0_scrape", { surface: "v0" }, flags, now)).toBe(+ false,+ );+ });++ it("treats a missing or non-string flag as no extension", () => {+ expect(hasSunsetExtension("v0_scrape", { surface: "v0" }, null, now)).toBe(+ false,+ );+ const numeric = { deprecationExtensions: { v0: 20270131 } } as any;+ expect(+ hasSunsetExtension("v0_scrape", { surface: "v0" }, numeric, now),+ ).toBe(false);+ });++ it.each([+ "next spring",+ "2027/01/31",+ "2027-02-30",+ "2027",+ "2027-01-31T12:00:00+02:00",+ "2027-01-31T25:00:00Z",+ ])("treats %s as no extension", until => {+ expect(+ hasSunsetExtension(+ "v0_scrape",+ { surface: "v0" },+ extendedUntil(until, "v0"),+ now,+ ),+ ).toBe(false);+ });+});++describe("deprecationMiddleware sunset gate", () => {+ it("serves the route with a Sunset header before the sunset", async () => {+ vi.setSystemTime(BEFORE_SUNSET);+ const res = await request(appFor(SUNSET_KEY, { flags: null })).get(+ "/probe",+ );++ expect(res.status).toBe(200);+ expect(res.headers["sunset"]).toBe("Tue, 03 Nov 2026 23:59:59 GMT");+ expect(res.body.warnings).toHaveLength(1);+ });++ it("answers 410 after the sunset", async () => {+ vi.setSystemTime(AFTER_SUNSET);+ const res = await request(appFor(SUNSET_KEY, { flags: null })).get(+ "/probe",+ );++ expect(res.status).toBe(410);+ expect(res.body).toMatchObject({+ success: false,+ code: "ENDPOINT_REMOVED",+ replacement: "/v2/search/developer",+ });+ expect(res.headers["sunset"]).toBe("Tue, 03 Nov 2026 23:59:59 GMT");+ });++ it("keeps serving a team whose extension runs later", async () => {+ vi.setSystemTime(AFTER_SUNSET);+ const res = await request(+ appFor(SUNSET_KEY, { flags: extendedUntil("2027-01-31T00:00:00Z") }),+ ).get("/probe");++ expect(res.status).toBe(200);+ expect(res.body.success).toBe(true);+ expect(res.body.warnings).toHaveLength(1);+ expect(mocks.getApiKeyTeamFlags).not.toHaveBeenCalled();+ });++ it("answers 410 once a team's extension has run out", async () => {+ vi.setSystemTime(AFTER_SUNSET);+ const res = await request(+ appFor(SUNSET_KEY, { flags: extendedUntil("2026-11-05T00:00:00Z") }),+ ).get("/probe");++ expect(res.status).toBe(410);+ });++ it("looks the team up by API key on routes that authenticate later", async () => {+ vi.setSystemTime(AFTER_SUNSET);+ mocks.getApiKeyTeamFlags.mockResolvedValueOnce(+ extendedUntil("2027-01-31T00:00:00Z"),+ );+ const extended = await request(appFor(SUNSET_KEY)).get("/probe");++ expect(extended.status).toBe(200);+ expect(mocks.getApiKeyTeamFlags).toHaveBeenCalledTimes(1);++ mocks.getApiKeyTeamFlags.mockResolvedValueOnce(null);+ const unknown = await request(appFor(SUNSET_KEY)).get("/probe");++ expect(unknown.status).toBe(410);+ });++ it("passes a failed lookup on as an error, not a 410", async () => {+ vi.setSystemTime(AFTER_SUNSET);+ mocks.getApiKeyTeamFlags.mockRejectedValueOnce(new Error("lookup down"));+ const res = await request(appFor(SUNSET_KEY)).get("/probe");++ expect(res.status).toBe(500);+ });++ it("never looks anything up for an entry without a sunset", async () => {+ vi.setSystemTime(AFTER_SUNSET);+ const res = await request(appFor("v0_scrape")).get("/probe");++ expect(res.status).toBe(200);+ expect(res.headers["sunset"]).toBeUndefined();+ expect(mocks.getApiKeyTeamFlags).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex dfce42515..e1495c4a9 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -44,7 +44,11 @@ import {AuthCreditUsageChunkRow,} from "../db/rpc";import { AuthResponse, RateLimiterMode } from "../types";-import { AuthCreditUsageChunk, AuthCreditUsageChunkFromTeam } from "./v1/types";+import {+ AuthCreditUsageChunk,+ AuthCreditUsageChunkFromTeam,+ TeamFlags,+} from "./v1/types";import {FIRECRAWL_REST_RESOURCE,OAuthIntrospectionUnavailableError,@@ -513,6 +517,21 @@ export async function getACUCTeam(}}+/**+ * The flags of the team behind a request's plain API key, read without rate+ * limiting or the other auth checks. Null without a known API key.+ */+export async function getApiKeyTeamFlags(req: {+ headers: { authorization?: string };+}): Promise<TeamFlags | null> {+ const token = req.headers.authorization?.split(" ")[1];+ if (!token) return null;+ const normalizedApi = parseApi(token);+ if (!normalizedApiIsUuid(normalizedApi)) return null;+ const chunk = await getACUC(normalizedApi);+ return chunk?.flags ?? null;+}+export async function clearACUC(api_key: string): Promise<void> {// Clear both purpose-qualified keys and the legacy unqualified keys during// rollout so credential-purpose isolation cannot leave stale auth entries.diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex a71657a35..3ad00e9cd 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1417,6 +1417,9 @@ export type TeamFlags = {// Every request is processed by US-based vendors only. A route with no US// vendor is refused or skipped with a warning.usOnly?: boolean;+ // Per-team extensions past a deprecation's sunset (lib/deprecations.ts):+ // surface or entry key to the YYYY-MM-DD or UTC timestamp it runs until.+ deprecationExtensions?: Record<string, string>;organizationDataSourceAccess?: Record<string,{diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 9474f38b1..7680dc445 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -2192,6 +2192,9 @@ export type TeamFlags = {// Every request is processed by US-based vendors only. A route with no US// vendor is refused or skipped with a warning.usOnly?: boolean;+ // Per-team extensions past a deprecation's sunset (lib/deprecations.ts):+ // surface or entry key to the YYYY-MM-DD or UTC timestamp it runs until.+ deprecationExtensions?: Record<string, string>;organizationDataSourceAccess?: Record<string,{diff --git a/apps/api/src/lib/deprecations.ts b/apps/api/src/lib/deprecations.tsindex f74a17553..0745eacbd 100644--- a/apps/api/src/lib/deprecations.ts+++ b/apps/api/src/lib/deprecations.ts@@ -2,14 +2,21 @@ import { NextFunction, Request, Response } from "express";import { logger } from "./logger";import { requestOrigin } from "./request-origin";import { resolveIntegration } from "../utils/integration";+import { getApiKeyTeamFlags } from "../controllers/auth";+import type { RequestWithMaybeACUC, TeamFlags } from "../controllers/v1/types";interface Deprecation {message: string;replacement?: string;// RFC 9745 requires a Date here, e.g. "@1788393600".deprecatedAt: string;+ // RFC 8594 IMF-fixdate. From then on deprecationMiddleware answers 410,+ // except for a team whose deprecationExtensions flag runs later.sunset?: string;docs?: string;+ // Key of the team extension flag, shared by every entry of one surface.+ // Defaults to the entry's own key.+ surface?: string;}export interface Notice {@@ -74,28 +81,33 @@ const DEPRECATIONS = {message: "/v0/scrape is deprecated. Use /v2/scrape instead.",replacement: "/v2/scrape",deprecatedAt: "@1778025600",+ surface: "v0",},v0_crawl: {message: "/v0/crawl is deprecated. Use /v2/crawl instead.",replacement: "/v2/crawl",deprecatedAt: "@1778025600",+ surface: "v0",},v0_crawl_status: {message:"/v0/crawl/status/:jobId is deprecated. Use /v2/crawl/:jobId instead.",replacement: "/v2/crawl/:jobId",deprecatedAt: "@1778025600",+ surface: "v0",},v0_crawl_cancel: {message:"/v0/crawl/cancel/:jobId is deprecated. Use DELETE /v2/crawl/:jobId instead.",replacement: "/v2/crawl/:jobId",deprecatedAt: "@1778025600",+ surface: "v0",},v0_search: {message: "/v0/search is deprecated. Use /v2/search instead.",replacement: "/v2/search",deprecatedAt: "@1778025600",+ surface: "v0",},} as const satisfies Record<string, Deprecation>;@@ -145,45 +157,108 @@ function setDeprecationHeaders(res: Response, dep: Deprecation) {if (links.length) res.setHeader("Link", links.join(", "));}+/**+ * The instant a sunset takes effect, Infinity without one. Throws unless the+ * value is an exact IMF-fixdate, e.g. "Tue, 03 Nov 2026 23:59:59 GMT".+ */+export function sunsetInstant(key: string, sunset: string | undefined): number {+ if (sunset === undefined) return Infinity;+ const at = Date.parse(sunset);+ if (Number.isNaN(at) || new Date(at).toUTCString() !== sunset) {+ throw new Error(`Deprecation ${key} has an unreadable sunset: ${sunset}`);+ }+ return at;+}++const EXTENSION_UNTIL = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d{3})?Z)?$/;++/**+ * Whether a team's deprecationExtensions flag keeps a sunset route working for+ * it now. Anything but an exact YYYY-MM-DD or UTC timestamp counts as none.+ */+export function hasSunsetExtension(+ key: string,+ dep: Pick<Deprecation, "surface">,+ flags: TeamFlags | null | undefined,+ now = Date.now(),+): boolean {+ const until = flags?.deprecationExtensions?.[dep.surface ?? key];+ if (typeof until !== "string" || !EXTENSION_UNTIL.test(until)) return false;+ const at = Date.parse(until);+ return (+ !Number.isNaN(at) &&+ new Date(at).toISOString().startsWith(until.replace(/Z$/, "")) &&+ now < at+ );+}++/**+ * Mount after authMiddleware where the route has one: past sunset it reads the+ * team's extension from req.acuc, so keyless callers still get 401 first.+ */export function deprecationMiddleware(key: DeprecationKey) {const dep: Deprecation = DEPRECATIONS[key];- return (req: Request, res: Response, next: NextFunction) => {+ const sunsetAt = sunsetInstant(key, dep.sunset);++ const annotate = (res: Response) => {setDeprecationHeaders(res, dep);applyNotice(res, { message: dep.message, replacement: dep.replacement });- next();+ };++ return (req: RequestWithMaybeACUC, res: Response, next: NextFunction) => {+ if (Date.now() < sunsetAt) {+ annotate(res);+ return next();+ }++ // v0 controllers authenticate themselves, so their routes have no acuc yet.+ const lookup = req.acuc+ ? Promise.resolve(req.acuc.flags)+ : getApiKeyTeamFlags(req);+ lookup+ .then(flags => {+ if (!hasSunsetExtension(key, dep, flags)) {+ return respondRemoved(req, res, key);+ }+ annotate(res);+ next();+ })+ .catch(next);};}-// For an endpoint past its sunset. Mount it ahead of auth, so every caller gets-// the pointer and nothing is rate limited, credit checked or billed.-export function removedEndpointHandler(key: DeprecationKey) {+function respondRemoved(req: Request, res: Response, key: DeprecationKey) {const dep: Deprecation = DEPRECATIONS[key];- return (req: Request, res: Response) => {- const { origin, integration } = req.query;- logger.warn("Removed endpoint called", {- module: "deprecations",- deprecation: key,- path: req.baseUrl + req.path,- origin: requestOrigin(- { origin: typeof origin === "string" ? origin : undefined },+ const { origin, integration } = req.query;+ logger.warn("Removed endpoint called", {+ module: "deprecations",+ deprecation: key,+ path: req.baseUrl + req.path,+ origin: requestOrigin(+ { origin: typeof origin === "string" ? origin : undefined },+ req,+ ),+ integration:+ resolveIntegration(+ typeof integration === "string" ? integration : undefined,req,- ),- integration:- resolveIntegration(- typeof integration === "string" ? integration : undefined,- req,- ) ?? null,- userAgent: req.headers["user-agent"],- });+ ) ?? null,+ userAgent: req.headers["user-agent"],+ });- setDeprecationHeaders(res, dep);- res.status(410).json({- success: false,- code: "ENDPOINT_REMOVED",- error: dep.message,- detail: dep.message,- replacement: dep.replacement,- documentation_url: dep.docs,- });- };+ setDeprecationHeaders(res, dep);+ res.status(410).json({+ success: false,+ code: "ENDPOINT_REMOVED",+ error: dep.message,+ detail: dep.message,+ replacement: dep.replacement,+ documentation_url: dep.docs,+ });+}++// For an endpoint whose code is gone. Mount it ahead of auth, so every caller+// gets the pointer and nothing is rate limited, credit checked or billed.+export function removedEndpointHandler(key: DeprecationKey) {+ return (req: Request, res: Response) => respondRemoved(req, res, key);}