Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): sunset dates and per-team extensions for deprecated routes (#5605)

Gergő Móriczbf6739d95cac
diff --git a/apps/api/src/__tests__/routes/deprecations-sunset.routes.test.ts b/apps/api/src/__tests__/routes/deprecations-sunset.routes.test.ts
new file mode 100644
index 000000000..67e325e13
--- /dev/null
+++ b/apps/api/src/__tests__/routes/deprecations-sunset.routes.test.ts
@@ -0,0 +1,230 @@
+import express from "express";
+import request from "supertest";
+
+const mocks = vi.hoisted(() => ({ getApiKeyTeamFlags: vi.fn() }));
+
+vi.mock("../../controllers/auth", () => ({
+ getApiKeyTeamFlags: mocks.getApiKeyTeamFlags,
+}));
+
+import {
+ deprecationMiddleware,
+ hasSunsetExtension,
+ sunsetInstant,
+} from "../../lib/deprecations";
+
+// The only entry with a sunset today: Tue, 03 Nov 2026 23:59:59 GMT.
+const SUNSET_KEY = "v2_research_github_search";
+const BEFORE_SUNSET = new Date("2026-11-01T00:00:00Z");
+const AFTER_SUNSET = new Date("2026-11-10T00:00:00Z");
+
+function appFor(
+ key: Parameters<typeof deprecationMiddleware>[0],
+ acuc?: { flags: unknown },
+) {
+ const app = express();
+ app.get(
+ "/probe",
+ (req: any, _res, next) => {
+ if (acuc) req.acuc = acuc;
+ next();
+ },
+ deprecationMiddleware(key),
+ (_req, res) => {
+ res.status(200).json({ success: true });
+ },
+ );
+ return app;
+}
+
+function extendedUntil(until: string, surface = SUNSET_KEY) {
+ return { deprecationExtensions: { [surface]: until } };
+}
+
+beforeEach(() => {
+ vi.useFakeTimers({ toFake: ["Date"] });
+ mocks.getApiKeyTeamFlags.mockReset();
+});
+
+afterEach(() => {
+ vi.useRealTimers();
+});
+
+describe("sunsetInstant", () => {
+ it("reads an exact IMF-fixdate", () => {
+ expect(sunsetInstant("k", "Tue, 03 Nov 2026 23:59:59 GMT")).toBe(
+ Date.parse("2026-11-03T23:59:59Z"),
+ );
+ });
+
+ it("is never reached without a sunset", () => {
+ expect(sunsetInstant("k", undefined)).toBe(Infinity);
+ });
+
+ it.each([
+ "Tue, 31 Nov 2026 23:59:59 GMT",
+ "Mon, 03 Nov 2026 23:59:59 GMT",
+ "3 Nov 2026",
+ "2026-11-03",
+ "soon",
+ ])("refuses %s", sunset => {
+ expect(() => sunsetInstant("k", sunset)).toThrow(/unreadable sunset/);
+ });
+});
+
+describe("hasSunsetExtension", () => {
+ const now = Date.parse("2026-12-01T00:00:00Z");
+
+ it("reads the flag under the entry's surface", () => {
+ expect(
+ hasSunsetExtension(
+ "v0_scrape",
+ { surface: "v0" },
+ extendedUntil("2027-01-31", "v0"),
+ now,
+ ),
+ ).toBe(true);
+ });
+
+ it("ignores a flag under the entry key when the entry has a surface", () => {
+ expect(
+ hasSunsetExtension(
+ "v0_scrape",
+ { surface: "v0" },
+ extendedUntil("2027-01-31", "v0_scrape"),
+ now,
+ ),
+ ).toBe(false);
+ });
+
+ it("falls back to the entry key without a surface", () => {
+ expect(
+ hasSunsetExtension(
+ "v1_llmstxt",
+ {},
+ extendedUntil("2027-01-31", "v1_llmstxt"),
+ now,
+ ),
+ ).toBe(true);
+ });
+
+ it("ends at the flag's instant", () => {
+ const flags = extendedUntil("2026-12-01T00:00:00Z", "v0");
+ expect(
+ hasSunsetExtension("v0_scrape", { surface: "v0" }, flags, now - 1),
+ ).toBe(true);
+ expect(hasSunsetExtension("v0_scrape", { surface: "v0" }, flags, now)).toBe(
+ false,
+ );
+ });
+
+ it("treats a missing or non-string flag as no extension", () => {
+ expect(hasSunsetExtension("v0_scrape", { surface: "v0" }, null, now)).toBe(
+ false,
+ );
+ const numeric = { deprecationExtensions: { v0: 20270131 } } as any;
+ expect(
+ hasSunsetExtension("v0_scrape", { surface: "v0" }, numeric, now),
+ ).toBe(false);
+ });
+
+ it.each([
+ "next spring",
+ "2027/01/31",
+ "2027-02-30",
+ "2027",
+ "2027-01-31T12:00:00+02:00",
+ "2027-01-31T25:00:00Z",
+ ])("treats %s as no extension", until => {
+ expect(
+ hasSunsetExtension(
+ "v0_scrape",
+ { surface: "v0" },
+ extendedUntil(until, "v0"),
+ now,
+ ),
+ ).toBe(false);
+ });
+});
+
+describe("deprecationMiddleware sunset gate", () => {
+ it("serves the route with a Sunset header before the sunset", async () => {
+ vi.setSystemTime(BEFORE_SUNSET);
+ const res = await request(appFor(SUNSET_KEY, { flags: null })).get(
+ "/probe",
+ );
+
+ expect(res.status).toBe(200);
+ expect(res.headers["sunset"]).toBe("Tue, 03 Nov 2026 23:59:59 GMT");
+ expect(res.body.warnings).toHaveLength(1);
+ });
+
+ it("answers 410 after the sunset", async () => {
+ vi.setSystemTime(AFTER_SUNSET);
+ const res = await request(appFor(SUNSET_KEY, { flags: null })).get(
+ "/probe",
+ );
+
+ expect(res.status).toBe(410);
+ expect(res.body).toMatchObject({
+ success: false,
+ code: "ENDPOINT_REMOVED",
+ replacement: "/v2/search/developer",
+ });
+ expect(res.headers["sunset"]).toBe("Tue, 03 Nov 2026 23:59:59 GMT");
+ });
+
+ it("keeps serving a team whose extension runs later", async () => {
+ vi.setSystemTime(AFTER_SUNSET);
+ const res = await request(
+ appFor(SUNSET_KEY, { flags: extendedUntil("2027-01-31T00:00:00Z") }),
+ ).get("/probe");
+
+ expect(res.status).toBe(200);
+ expect(res.body.success).toBe(true);
+ expect(res.body.warnings).toHaveLength(1);
+ expect(mocks.getApiKeyTeamFlags).not.toHaveBeenCalled();
+ });
+
+ it("answers 410 once a team's extension has run out", async () => {
+ vi.setSystemTime(AFTER_SUNSET);
+ const res = await request(
+ appFor(SUNSET_KEY, { flags: extendedUntil("2026-11-05T00:00:00Z") }),
+ ).get("/probe");
+
+ expect(res.status).toBe(410);
+ });
+
+ it("looks the team up by API key on routes that authenticate later", async () => {
+ vi.setSystemTime(AFTER_SUNSET);
+ mocks.getApiKeyTeamFlags.mockResolvedValueOnce(
+ extendedUntil("2027-01-31T00:00:00Z"),
+ );
+ const extended = await request(appFor(SUNSET_KEY)).get("/probe");
+
+ expect(extended.status).toBe(200);
+ expect(mocks.getApiKeyTeamFlags).toHaveBeenCalledTimes(1);
+
+ mocks.getApiKeyTeamFlags.mockResolvedValueOnce(null);
+ const unknown = await request(appFor(SUNSET_KEY)).get("/probe");
+
+ expect(unknown.status).toBe(410);
+ });
+
+ it("passes a failed lookup on as an error, not a 410", async () => {
+ vi.setSystemTime(AFTER_SUNSET);
+ mocks.getApiKeyTeamFlags.mockRejectedValueOnce(new Error("lookup down"));
+ const res = await request(appFor(SUNSET_KEY)).get("/probe");
+
+ expect(res.status).toBe(500);
+ });
+
+ it("never looks anything up for an entry without a sunset", async () => {
+ vi.setSystemTime(AFTER_SUNSET);
+ const res = await request(appFor("v0_scrape")).get("/probe");
+
+ expect(res.status).toBe(200);
+ expect(res.headers["sunset"]).toBeUndefined();
+ expect(mocks.getApiKeyTeamFlags).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index dfce42515..e1495c4a9 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -44,7 +44,11 @@ import {
AuthCreditUsageChunkRow,
} from "../db/rpc";
import { AuthResponse, RateLimiterMode } from "../types";
-import { AuthCreditUsageChunk, AuthCreditUsageChunkFromTeam } from "./v1/types";
+import {
+ AuthCreditUsageChunk,
+ AuthCreditUsageChunkFromTeam,
+ TeamFlags,
+} from "./v1/types";
import {
FIRECRAWL_REST_RESOURCE,
OAuthIntrospectionUnavailableError,
@@ -513,6 +517,21 @@ export async function getACUCTeam(
}
}
+/**
+ * The flags of the team behind a request's plain API key, read without rate
+ * limiting or the other auth checks. Null without a known API key.
+ */
+export async function getApiKeyTeamFlags(req: {
+ headers: { authorization?: string };
+}): Promise<TeamFlags | null> {
+ const token = req.headers.authorization?.split(" ")[1];
+ if (!token) return null;
+ const normalizedApi = parseApi(token);
+ if (!normalizedApiIsUuid(normalizedApi)) return null;
+ const chunk = await getACUC(normalizedApi);
+ return chunk?.flags ?? null;
+}
+
export async function clearACUC(api_key: string): Promise<void> {
// Clear both purpose-qualified keys and the legacy unqualified keys during
// rollout so credential-purpose isolation cannot leave stale auth entries.
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index a71657a35..3ad00e9cd 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1417,6 +1417,9 @@ export type TeamFlags = {
// Every request is processed by US-based vendors only. A route with no US
// vendor is refused or skipped with a warning.
usOnly?: boolean;
+ // Per-team extensions past a deprecation's sunset (lib/deprecations.ts):
+ // surface or entry key to the YYYY-MM-DD or UTC timestamp it runs until.
+ deprecationExtensions?: Record<string, string>;
organizationDataSourceAccess?: Record<
string,
{
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 9474f38b1..7680dc445 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -2192,6 +2192,9 @@ export type TeamFlags = {
// Every request is processed by US-based vendors only. A route with no US
// vendor is refused or skipped with a warning.
usOnly?: boolean;
+ // Per-team extensions past a deprecation's sunset (lib/deprecations.ts):
+ // surface or entry key to the YYYY-MM-DD or UTC timestamp it runs until.
+ deprecationExtensions?: Record<string, string>;
organizationDataSourceAccess?: Record<
string,
{
diff --git a/apps/api/src/lib/deprecations.ts b/apps/api/src/lib/deprecations.ts
index f74a17553..0745eacbd 100644
--- a/apps/api/src/lib/deprecations.ts
+++ b/apps/api/src/lib/deprecations.ts
@@ -2,14 +2,21 @@ import { NextFunction, Request, Response } from "express";
import { logger } from "./logger";
import { requestOrigin } from "./request-origin";
import { resolveIntegration } from "../utils/integration";
+import { getApiKeyTeamFlags } from "../controllers/auth";
+import type { RequestWithMaybeACUC, TeamFlags } from "../controllers/v1/types";
interface Deprecation {
message: string;
replacement?: string;
// RFC 9745 requires a Date here, e.g. "@1788393600".
deprecatedAt: string;
+ // RFC 8594 IMF-fixdate. From then on deprecationMiddleware answers 410,
+ // except for a team whose deprecationExtensions flag runs later.
sunset?: string;
docs?: string;
+ // Key of the team extension flag, shared by every entry of one surface.
+ // Defaults to the entry's own key.
+ surface?: string;
}
export interface Notice {
@@ -74,28 +81,33 @@ const DEPRECATIONS = {
message: "/v0/scrape is deprecated. Use /v2/scrape instead.",
replacement: "/v2/scrape",
deprecatedAt: "@1778025600",
+ surface: "v0",
},
v0_crawl: {
message: "/v0/crawl is deprecated. Use /v2/crawl instead.",
replacement: "/v2/crawl",
deprecatedAt: "@1778025600",
+ surface: "v0",
},
v0_crawl_status: {
message:
"/v0/crawl/status/:jobId is deprecated. Use /v2/crawl/:jobId instead.",
replacement: "/v2/crawl/:jobId",
deprecatedAt: "@1778025600",
+ surface: "v0",
},
v0_crawl_cancel: {
message:
"/v0/crawl/cancel/:jobId is deprecated. Use DELETE /v2/crawl/:jobId instead.",
replacement: "/v2/crawl/:jobId",
deprecatedAt: "@1778025600",
+ surface: "v0",
},
v0_search: {
message: "/v0/search is deprecated. Use /v2/search instead.",
replacement: "/v2/search",
deprecatedAt: "@1778025600",
+ surface: "v0",
},
} as const satisfies Record<string, Deprecation>;
@@ -145,45 +157,108 @@ function setDeprecationHeaders(res: Response, dep: Deprecation) {
if (links.length) res.setHeader("Link", links.join(", "));
}
+/**
+ * The instant a sunset takes effect, Infinity without one. Throws unless the
+ * value is an exact IMF-fixdate, e.g. "Tue, 03 Nov 2026 23:59:59 GMT".
+ */
+export function sunsetInstant(key: string, sunset: string | undefined): number {
+ if (sunset === undefined) return Infinity;
+ const at = Date.parse(sunset);
+ if (Number.isNaN(at) || new Date(at).toUTCString() !== sunset) {
+ throw new Error(`Deprecation ${key} has an unreadable sunset: ${sunset}`);
+ }
+ return at;
+}
+
+const EXTENSION_UNTIL = /^\d{4}-\d{2}-\d{2}(T\d{2}:\d{2}:\d{2}(\.\d{3})?Z)?$/;
+
+/**
+ * Whether a team's deprecationExtensions flag keeps a sunset route working for
+ * it now. Anything but an exact YYYY-MM-DD or UTC timestamp counts as none.
+ */
+export function hasSunsetExtension(
+ key: string,
+ dep: Pick<Deprecation, "surface">,
+ flags: TeamFlags | null | undefined,
+ now = Date.now(),
+): boolean {
+ const until = flags?.deprecationExtensions?.[dep.surface ?? key];
+ if (typeof until !== "string" || !EXTENSION_UNTIL.test(until)) return false;
+ const at = Date.parse(until);
+ return (
+ !Number.isNaN(at) &&
+ new Date(at).toISOString().startsWith(until.replace(/Z$/, "")) &&
+ now < at
+ );
+}
+
+/**
+ * Mount after authMiddleware where the route has one: past sunset it reads the
+ * team's extension from req.acuc, so keyless callers still get 401 first.
+ */
export function deprecationMiddleware(key: DeprecationKey) {
const dep: Deprecation = DEPRECATIONS[key];
- return (req: Request, res: Response, next: NextFunction) => {
+ const sunsetAt = sunsetInstant(key, dep.sunset);
+
+ const annotate = (res: Response) => {
setDeprecationHeaders(res, dep);
applyNotice(res, { message: dep.message, replacement: dep.replacement });
- next();
+ };
+
+ return (req: RequestWithMaybeACUC, res: Response, next: NextFunction) => {
+ if (Date.now() < sunsetAt) {
+ annotate(res);
+ return next();
+ }
+
+ // v0 controllers authenticate themselves, so their routes have no acuc yet.
+ const lookup = req.acuc
+ ? Promise.resolve(req.acuc.flags)
+ : getApiKeyTeamFlags(req);
+ lookup
+ .then(flags => {
+ if (!hasSunsetExtension(key, dep, flags)) {
+ return respondRemoved(req, res, key);
+ }
+ annotate(res);
+ next();
+ })
+ .catch(next);
};
}
-// For an endpoint past its sunset. Mount it ahead of auth, so every caller gets
-// the pointer and nothing is rate limited, credit checked or billed.
-export function removedEndpointHandler(key: DeprecationKey) {
+function respondRemoved(req: Request, res: Response, key: DeprecationKey) {
const dep: Deprecation = DEPRECATIONS[key];
- return (req: Request, res: Response) => {
- const { origin, integration } = req.query;
- logger.warn("Removed endpoint called", {
- module: "deprecations",
- deprecation: key,
- path: req.baseUrl + req.path,
- origin: requestOrigin(
- { origin: typeof origin === "string" ? origin : undefined },
+ const { origin, integration } = req.query;
+ logger.warn("Removed endpoint called", {
+ module: "deprecations",
+ deprecation: key,
+ path: req.baseUrl + req.path,
+ origin: requestOrigin(
+ { origin: typeof origin === "string" ? origin : undefined },
+ req,
+ ),
+ integration:
+ resolveIntegration(
+ typeof integration === "string" ? integration : undefined,
req,
- ),
- integration:
- resolveIntegration(
- typeof integration === "string" ? integration : undefined,
- req,
- ) ?? null,
- userAgent: req.headers["user-agent"],
- });
+ ) ?? null,
+ userAgent: req.headers["user-agent"],
+ });
- setDeprecationHeaders(res, dep);
- res.status(410).json({
- success: false,
- code: "ENDPOINT_REMOVED",
- error: dep.message,
- detail: dep.message,
- replacement: dep.replacement,
- documentation_url: dep.docs,
- });
- };
+ setDeprecationHeaders(res, dep);
+ res.status(410).json({
+ success: false,
+ code: "ENDPOINT_REMOVED",
+ error: dep.message,
+ detail: dep.message,
+ replacement: dep.replacement,
+ documentation_url: dep.docs,
+ });
+}
+
+// For an endpoint whose code is gone. Mount it ahead of auth, so every caller
+// gets the pointer and nothing is rate limited, credit checked or billed.
+export function removedEndpointHandler(key: DeprecationKey) {
+ return (req: Request, res: Response) => respondRemoved(req, res, key);
}