Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): retire research index GitHub search with 410 Gone (#5537)
Kris Oei5055e08ad87b
diff --git a/apps/api/src/__tests__/routes/research-credit-gate.routes.test.ts b/apps/api/src/__tests__/routes/research-credit-gate.routes.test.tsindex 7551d35ee..698338093 100644--- a/apps/api/src/__tests__/routes/research-credit-gate.routes.test.ts+++ b/apps/api/src/__tests__/routes/research-credit-gate.routes.test.ts@@ -73,7 +73,6 @@ beforeEach(() => {describe.each([["developer search", "/v2/search/developer?query=x"],["legacy developer search", "/v2/developer/search?query=x"],- ["GitHub research search", "/v2/search/research/github?query=x"],])("%s credit gate", (_name, path) => {it("refuses with 402 when the team can't cover a search", async () => {checkCredits.mockResolvedValue({ allowed: false, remaining: 0 });diff --git a/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.ts b/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.tsindex 2017c4eb2..401a08f97 100644--- a/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.ts+++ b/apps/api/src/__tests__/routes/research-github-deprecation.routes.test.ts@@ -1,44 +1,39 @@import express from "express";import request from "supertest";import { createResearchRouter } from "../../controllers/v2/research-proxy";+import { v2Router } from "../../routes/v2";-// Mounts the real research router against a stubbed upstream, so the-// deprecation contract is exercised through the actual controller rather than-// a fake. The one thing only this test can reach is the logged payload: on the-// canonical mount the controller responds with the same object it later writes-// to Postgres, so a mutating res.json would leak the notice into the-// research_github_searches row.-const upstreamCalls: string[] = [];const loggedRows: any[] = [];-let upstreamBody: any = {- success: true,- results: [- {- resultType: "github_history",- repo: "milvus-io/milvus",- url: "https://github.com/milvus-io/milvus/issues/1",- pageType: "issue",- number: 1,- snippet: "hybrid search",- contentMd: "# hybrid search",- segmentCount: 2,- scores: { rrf: 0.5 },- },- ],-};-let upstreamStatus = 200;+// The real v2 router with the research proxy configured. Auth is stubbed to+// answer 401, so a 410 can only come from ahead of auth, rate limiting, the+// credit gate, the upstream call and billing.+vi.mock("../../config", async importOriginal => {+ const actual = await importOriginal<typeof import("../../config")>();+ return {+ ...actual,+ config: { ...actual.config, RESEARCH_PROXY_URL: "http://research.test" },+ };+});++vi.mock("../../routes/shared", async importOriginal => {+ const actual = await importOriginal<typeof import("../../routes/shared")>();+ return {+ ...actual,+ authMiddleware: () => (_req: any, res: any) =>+ res.status(401).json({ success: false, error: "auth reached" }),+ };+});+// The paper routes run through the real research router against a stubbed+// upstream.vi.mock("../../lib/research-upstream", () => ({- fetchResearchUpstream: async (options: { path: string }) => {- upstreamCalls.push(options.path);- return {- status: upstreamStatus,- ok: upstreamStatus >= 200 && upstreamStatus < 300,- headers: new Headers(),- text: async () => JSON.stringify(upstreamBody),- };- },+ fetchResearchUpstream: async () => ({+ status: 200,+ ok: true,+ headers: new Headers(),+ text: async () => JSON.stringify({ success: true, results: [] }),+ }),}));vi.mock("../../services/logging/log_job", () => ({@@ -52,7 +47,6 @@ vi.mock("../../lib/keyless", () => ({chargeKeylessCredits: async () => {},}));-// These tests cover the deprecation notice, not the daily index limit.// The limiter needs Redis, so here it admits every call.vi.mock("../../lib/index-daily-limit", () => ({reserveIndexCalls: vi.fn(async () => ({ ok: true, release: async () => {} })),@@ -63,36 +57,42 @@ vi.mock("../../services/billing/credit_billing", () => ({billTeam: async () => {},}));-function appWith(legacy: boolean) {- const app = express();- app.use((req: any, _res, next) => {- req.auth = { team_id: "team-test", plan: "standard" };- req.acuc = { api_key_id: null };- next();- });- app.use(- legacy ? "/v2/research" : "/v2/search/research",- createResearchRouter(legacy ? { legacy: true } : {}),- );- return app;-}--const canonical = appWith(false);-const legacy = appWith(true);+const app = express();+app.use("/v2", v2Router);-beforeEach(() => {- upstreamCalls.length = 0;- loggedRows.length = 0;- upstreamStatus = 200;+const canonical = express();+canonical.use((req: any, _res, next) => {+ req.auth = { team_id: "team-test", plan: "standard" };+ req.acuc = { api_key_id: null };+ next();});--describe("github search deprecation, canonical mount", () => {- it("sets every deprecation header", async () => {- const res = await request(canonical).get(- "/v2/search/research/github?query=milvus+hybrid+search&k=3",+canonical.use("/v2/search/research", createResearchRouter());++describe.each([+ ["canonical", "/v2/search/research/github"],+ ["legacy", "/v2/research/github"],+])("removed github search, %s path", (_name, path) => {+ it("answers 410 with the developer index pointer before auth", async () => {+ const res = await request(app).get(+ `${path}?query=milvus+hybrid+search&k=3`,);- expect(res.statusCode).toBe(200);+ expect(res.statusCode).toBe(410);+ expect(res.body).toEqual({+ success: false,+ code: "ENDPOINT_REMOVED",+ error: expect.stringContaining("/v2/search/developer"),+ detail: res.body.error,+ replacement: "/v2/search/developer",+ documentation_url: "https://docs.firecrawl.dev/features/developer",+ });+ expect(res.body.error).toContain("removed after 2026-11-03");+ });++ it("keeps the deprecation headers", async () => {+ const res = await request(app).get(path);++ expect(res.statusCode).toBe(410);expect(res.headers["deprecation"]).toBe("@1788393600");expect(res.headers["sunset"]).toBe("Tue, 03 Nov 2026 23:59:59 GMT");expect(res.headers["link"]).toContain(@@ -101,92 +101,20 @@ describe("github search deprecation, canonical mount", () => {expect(res.headers["link"]).toContain('</v2/search/developer>; rel="successor-version"',);- expect(res.headers["warning"]).toMatch(/^299 - "/);- expect(res.headers["warning"]).toContain("2026-11-03");- });-- it("adds warnings and replacement to the body without losing the results", async () => {- const res = await request(canonical).get(- "/v2/search/research/github?query=x",- );-- expect(res.body.success).toBe(true);- expect(res.body.results).toHaveLength(1);- expect(res.body.results[0].repo).toBe("milvus-io/milvus");- expect(res.body.replacement).toBe("/v2/search/developer");- expect(res.body.warnings).toHaveLength(1);- expect(res.body.warnings[0]).toContain("/v2/search/developer");- expect(res.body.warnings[0]).toContain("2026-11-03");- });-- it("keeps the deprecation notice out of the logged row", async () => {- await request(canonical).get("/v2/search/research/github?query=x");-- expect(loggedRows).toHaveLength(1);- const logged = loggedRows[0].response;- expect(logged.success).toBe(true);- expect(logged.results).toHaveLength(1);- expect(logged.warnings).toBeUndefined();- expect(logged.replacement).toBeUndefined();- expect(JSON.stringify(logged)).not.toContain("deprecated");- });-- it("still proxies to the unchanged upstream path", async () => {- await request(canonical).get("/v2/search/research/github?query=x");-- expect(upstreamCalls).toEqual(["/v2/research/github"]);- });-- it("warns on a rejected request too", async () => {- const res = await request(canonical).get("/v2/search/research/github");-- expect(res.statusCode).toBe(400);- expect(res.body.success).toBe(false);- expect(res.headers["deprecation"]).toBe("@1788393600");- expect(res.body.warnings).toHaveLength(1);- });-- it("leaves the paper routes on the same router alone", async () => {- const res = await request(canonical).get(- "/v2/search/research/papers?query=rag",- );-- expect(res.statusCode).toBe(200);- expect(res.headers["deprecation"]).toBeUndefined();- expect(res.headers["sunset"]).toBeUndefined();- expect(res.headers["link"]).toBeUndefined();- expect(res.headers["warning"]).toBeUndefined();- expect(res.body.warnings).toBeUndefined();- expect(res.body.replacement).toBeUndefined();- expect(loggedRows[0].response.warnings).toBeUndefined();});});-describe("github search deprecation, legacy mount", () => {- it("keeps the snake_case aliases alongside the notice", async () => {- const res = await request(legacy).get("/v2/research/github?query=x");-- expect(res.statusCode).toBe(200);- expect(res.headers["deprecation"]).toBe("@1788393600");- expect(res.body.results[0].result_type).toBe("github_history");- expect(res.body.results[0].page_type).toBe("issue");- expect(res.body.results[0].content_md).toBe("# hybrid search");- expect(res.body.results[0].segment_count).toBe(2);- expect(res.body.warnings).toHaveLength(1);- expect(res.body.replacement).toBe("/v2/search/developer");- });-- it("does not grow a snake_case twin of the injected keys", async () => {- const res = await request(legacy).get("/v2/research/github?query=x");-- expect(res.body).not.toHaveProperty("warnings_");- expect(res.body).not.toHaveProperty("replacement_");- });-- it("keeps the notice out of the logged row here as well", async () => {- await request(legacy).get("/v2/research/github?query=x");+it("leaves the paper routes on the same router alone", async () => {+ const res = await request(canonical).get(+ "/v2/search/research/papers?query=rag",+ );- expect(loggedRows[0].response.warnings).toBeUndefined();- expect(loggedRows[0].response.replacement).toBeUndefined();- });+ expect(res.statusCode).toBe(200);+ expect(res.headers["deprecation"]).toBeUndefined();+ expect(res.headers["sunset"]).toBeUndefined();+ expect(res.headers["link"]).toBeUndefined();+ expect(res.headers["warning"]).toBeUndefined();+ expect(res.body.warnings).toBeUndefined();+ expect(res.body.replacement).toBeUndefined();+ expect(loggedRows[0].response.warnings).toBeUndefined();});diff --git a/apps/api/src/__tests__/routes/research-keyless.routes.test.ts b/apps/api/src/__tests__/routes/research-keyless.routes.test.tsindex 26b0097ec..91712b482 100644--- a/apps/api/src/__tests__/routes/research-keyless.routes.test.ts+++ b/apps/api/src/__tests__/routes/research-keyless.routes.test.ts@@ -37,7 +37,6 @@ async function appWithResearchGate(flag: string | undefined) {router.get("/papers", reached);router.get("/papers/:id/similar", reached);router.get("/papers/:id", reached);- router.get("/github", reached);app.use("/v2/search/research",@@ -112,20 +111,6 @@ describe("RESEARCH_KEYLESS_DISABLED parsing", () => {expect(config.RESEARCH_KEYLESS_DISABLED).toEqual(["inspect", "similar"]);});- it("accepts the deprecated github operation only when it is named", async () => {- const { config } = await loadWithFlag("github");-- expect(config.RESEARCH_KEYLESS_DISABLED).toEqual(["github"]);- });-- it("keeps github out of the true/all shorthand and out of the default", async () => {- for (const flag of [undefined, "true", "all", "on", "1", "yes"]) {- const { config } = await loadWithFlag(flag);-- expect(config.RESEARCH_KEYLESS_DISABLED).not.toContain("github");- }- });-it("refuses an unknown operation at boot instead of silently ignoring it", async () => {await expect(loadWithFlag("inspect,papers")).rejects.toThrow();});@@ -156,33 +141,6 @@ describe("Research Index paper operation gate", () => {).toBe(false);});- it("blocks the deprecated github operation once the flag names it", async () => {- const { isResearchKeylessDisabled } = await loadWithFlag("github");-- expect(- isResearchKeylessDisabled(fakeRequest("/github", { query: "x" })),- ).toBe(true);- expect(- isResearchKeylessDisabled(- fakeRequest("/v2/search/research/github", { query: "x" }),- ),- ).toBe(true);- expect(- isResearchKeylessDisabled(fakeRequest("/papers", { query: "rag" })),- ).toBe(false);- });-- it("does not mistake a paper path for the github route", async () => {- const { isResearchKeylessDisabled } = await loadWithFlag("github");-- expect(isResearchKeylessDisabled(fakeRequest("/papers/github"))).toBe(- false,- );- expect(- isResearchKeylessDisabled(fakeRequest("/papers/github/similar")),- ).toBe(false);- });-it("blocks only the named operation when the scope is narrowed", async () => {const { isResearchKeylessDisabled } = await loadWithFlag("inspect");@@ -253,12 +211,4 @@ describe("keyless Research Index requests through the auth middleware", () => {);expect(response.body.reachedController).toBeUndefined();});-- it("leaves the non-paper github operation keyless", async () => {- const response = await request(app).get("/v2/search/research/github?q=x");-- expect(response.body?.error ?? "").not.toContain(- "not supported by the keyless free tier",- );- });});diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.tsindex 6ad20c6ea..359dc64c8 100644--- a/apps/api/src/__tests__/snips/v2/research.test.ts+++ b/apps/api/src/__tests__/snips/v2/research.test.ts@@ -531,21 +531,16 @@ describeIf(HAS_RESEARCH)("Research API", () => {const GITHUB_DEPRECATED_AT = "@1788393600";const GITHUB_SUNSET = "Tue, 03 Nov 2026 23:59:59 GMT";-describeIf(HAS_RESEARCH)("Research API github search deprecation", () => {- it("carries the deprecation headers and body warning on the canonical mount", async () => {- const identity = await idmux({- name: "research/github deprecation canonical",- credits: 100,+describeIf(HAS_RESEARCH)("Research API github search removal", () => {+ it("answers 410 with the deprecation headers on the canonical mount", async () => {+ const res = await researchRaw("/v2/search/research/github", {+ query: "milvus hybrid search",+ k: 3,});- const res = await researchRaw(- "/v2/search/research/github",- { query: "milvus hybrid search", k: 3 },- identity,- );-- expect(res.statusCode).toBe(200);- expect(res.body.success).toBe(true);+ expect(res.statusCode).toBe(410);+ expect(res.body.success).toBe(false);+ expect(res.body.code).toBe("ENDPOINT_REMOVED");expect(res.headers["deprecation"]).toBe(GITHUB_DEPRECATED_AT);expect(res.headers["sunset"]).toBe(GITHUB_SUNSET);@@ -553,12 +548,9 @@ describeIf(HAS_RESEARCH)("Research API github search deprecation", () => {expect(res.headers["link"]).toContain('</v2/search/developer>; rel="successor-version"',);- expect(res.headers["warning"]).toMatch(/^299 - "/);- expect(Array.isArray(res.body.warnings)).toBe(true);- expect(- res.body.warnings.some((w: string) => /\/v2\/search\/developer/.test(w)),- ).toBe(true);+ expect(res.body.error).toContain("/v2/search/developer");+ expect(res.body.detail).toBe(res.body.error);expect(res.body.replacement).toBe("/v2/search/developer");}, 120000);@@ -581,40 +573,23 @@ describeIf(HAS_RESEARCH)("Research API github search deprecation", () => {expect(res.body.replacement).toBeUndefined();}, 120000);- it("keeps the legacy snake_case aliases alongside the new warning", async () => {- const identity = await idmux({- name: "research/github deprecation legacy",- credits: 100,+ it("answers 410 on the legacy mount too", async () => {+ const res = await researchRaw("/v2/research/github", {+ query: "milvus hybrid search",+ k: 3,});- const res = await researchRaw(- "/v2/research/github",- { query: "milvus hybrid search", k: 3 },- identity,- );-- expect(res.statusCode).toBe(200);+ expect(res.statusCode).toBe(410);expect(res.headers["deprecation"]).toBe(GITHUB_DEPRECATED_AT);- expect(Array.isArray(res.body.warnings)).toBe(true);- expect(res.body.results.length).toBeGreaterThan(0);- // resultType is optional, so only assert the alias when the camel key is set.- const first = res.body.results[0];- if (first.resultType !== undefined) {- expect(first.result_type).toBe(first.resultType);- }+ expect(res.body.replacement).toBe("/v2/search/developer");}, 120000);- it("still warns when the request is rejected", async () => {- const identity = await idmux({- name: "research/github deprecation 400",- credits: 100,- });+ it("answers 410 without an API key or a query", async () => {+ const res = await researchRaw("/v2/search/research/github", {});- const res = await researchRaw("/v2/search/research/github", {}, identity);-- expect(res.statusCode).toBe(400);+ expect(res.statusCode).toBe(410);expect(res.body.success).toBe(false);expect(res.headers["deprecation"]).toBe(GITHUB_DEPRECATED_AT);- expect(Array.isArray(res.body.warnings)).toBe(true);+ expect(res.body.replacement).toBe("/v2/search/developer");}, 60000);});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex ee24e7e5e..935f844d4 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -48,16 +48,8 @@ const RESEARCH_PAPER_OPERATIONS = ["similar",] as const;-// "github" is out of the default and the true/all shorthand, so keyless-// behaviour is unchanged until someone names it. An explicit list replaces the-// default, so closing it means RESEARCH_KEYLESS_DISABLED=search,inspect,read,similar,github-const RESEARCH_KEYLESS_OPERATIONS = [- ...RESEARCH_PAPER_OPERATIONS,- "github",-] as const;-export type ResearchKeylessOperation =- (typeof RESEARCH_KEYLESS_OPERATIONS)[number];+ (typeof RESEARCH_PAPER_OPERATIONS)[number];const researchKeylessDisabled = z.preprocess(value => {@@ -74,7 +66,7 @@ const researchKeylessDisabled = z.preprocess(.filter(Boolean);},z- .array(z.enum(RESEARCH_KEYLESS_OPERATIONS))+ .array(z.enum(RESEARCH_PAPER_OPERATIONS)).default([...RESEARCH_PAPER_OPERATIONS]),);diff --git a/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts b/apps/api/src/controllers/v2/__tests__/research-timeouts.test.tsindex 9a961c6b3..dfadafae0 100644--- a/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts+++ b/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts@@ -124,14 +124,6 @@ describe("research upstream timeouts", () => {params: { id: "paper-1" },timeoutMs: 10_000,},- {- name: "GitHub search",- router: () => createResearchRouter(),- path: "/github",- query: { query: "retry logic" },- params: {},- timeoutMs: 12_000,- },{name: "developer search",router: () => createDeveloperRouter(),@@ -185,12 +177,12 @@ describe("research upstream timeouts", () => {])("maps a $name timeout to 504", async ({ error }) => {mocks.fetchResearchUpstream.mockRejectedValue(error);- const res = await callRoute(createResearchRouter(), "/github", {- query: "retry logic",+ const res = await callRoute(createDeveloperRouter(), "/search", {+ query: "http client",});expect(mocks.fetchResearchUpstream).toHaveBeenCalledWith(- expect.objectContaining({ timeoutMs: 12_000 }),+ expect.objectContaining({ timeoutMs: 15_000 }),);expect(res.status).toHaveBeenCalledWith(504);expect(res.end).toHaveBeenCalled();diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 51e9c197d..b38f03cb1 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -28,13 +28,12 @@ import type {import type { RequestWithAuth } from "../v1/types";import { checkCreditsMiddleware, wrap } from "../../routes/shared";import { SEARCH_CREDITS_FEATURE_ID } from "../../services/autumn/autumn.service";-import { deprecationMiddleware } from "../../lib/deprecations";import { integrationSchema, resolveIntegration } from "../../utils/integration";import { requestOrigin } from "../../lib/request-origin";const SEARCH_CREDITS_PER_TEN_RESULTS = 2;-// Billed developer and GitHub searches are refused up front when the team-// can't cover the smallest search charge, the same gate /v2/search applies.+// Billed developer searches are refused up front when the team can't cover+// the smallest search charge, the same gate /v2/search applies.const searchCreditGate = () =>checkCreditsMiddleware(SEARCH_CREDITS_PER_TEN_RESULTS,@@ -48,11 +47,10 @@ const GOV_SEARCH_TIMEOUT_MS = 15_000;const PAPER_SEARCH_TIMEOUT_MS = 30_000;const PAPER_INSPECT_TIMEOUT_MS = 5_000;const SIMILAR_PAPERS_TIMEOUT_MS = 10_000;-const GITHUB_SEARCH_TIMEOUT_MS = 12_000;const PAPER_READ_TIMEOUT_MS = 120_000;// The paper endpoints, all served by the Research Index and limited by its-// daily budget. GitHub search is billed per search and stays outside it.+// daily budget.const RESEARCH_INDEX_KINDS = new Set<ResearchRequestKind>(["research_paper_search","research_related_papers",@@ -117,12 +115,6 @@ const similarPapersSchema = z.strictObject({...commonQuery,});-const githubSearchSchema = z.strictObject({- query: z.string().min(1),- k: kSchema(100),- ...commonQuery,-});-const starsSchema = z.coerce.number().int().nonnegative().optional();const booleanFlag = z@@ -283,8 +275,8 @@ function creditsFor(body: any,req: RequestWithAuth<any, any, any>,) {- // The Research Index and the Government Index are free. GitHub search and- // developer/code search remain billable.+ // The Research Index and the Government Index are free. Developer/code+ // search remains billable.if (config.kind === "gov_search" || RESEARCH_INDEX_KINDS.has(config.kind))return 0;@@ -647,29 +639,6 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) {}),);- // On the route, not the mounts, so the paper routes stay undeprecated.- router.get(- "/github",- deprecationMiddleware("v2_research_github_search"),- searchCreditGate(),- wrap(- createResearchController(- githubSearchSchema,- ["query", "k"],- {- kind: "research_github_search",- table: "research_github_searches",- action: "searchGithub",- targetHint: params => String(params.query),- upstreamPath: () => "/v2/research/github",- timeoutMs: GITHUB_SEARCH_TIMEOUT_MS,- billAs: "search",- },- options,- ),- ),- );-return router;}diff --git a/apps/api/src/lib/deprecations.ts b/apps/api/src/lib/deprecations.tsindex 5cb4489f3..16881a91c 100644--- a/apps/api/src/lib/deprecations.ts+++ b/apps/api/src/lib/deprecations.ts@@ -1,4 +1,7 @@import { NextFunction, Request, Response } from "express";+import { logger } from "./logger";+import { requestOrigin } from "./request-origin";+import { resolveIntegration } from "../utils/integration";interface Deprecation {message: string;@@ -43,7 +46,7 @@ const DEPRECATIONS = {},v2_research_github_search: {message:- "The research index GitHub search (GET /v2/search/research/github, legacy GET /v2/research/github) is deprecated and stops responding after 2026-11-03. Use GET or POST /v2/search/developer instead: it searches GitHub issues, pull requests and READMEs plus curated documentation sources, returns matched passages, and adds filters for repo, language, license and stars. Response changes: 'snippet' becomes 'passages', results gain an 'id', and there is no score breakdown and no web fallback result type.",+ "The research index GitHub search (GET /v2/search/research/github, legacy GET /v2/research/github) was removed after 2026-11-03. Send the same query and k to GET or POST /v2/search/developer instead. See https://docs.firecrawl.dev/features/developer.",replacement: "/v2/search/developer",docs: "https://docs.firecrawl.dev/features/developer",deprecatedAt: "@1788393600",@@ -126,26 +129,61 @@ export function applyNotice(res: Response, notice: Notice) {};}+function setDeprecationHeaders(res: Response, dep: Deprecation) {+ // RFC 9745 Deprecation header.+ res.setHeader("Deprecation", dep.deprecatedAt);+ // RFC 8594 Sunset header.+ if (dep.sunset) res.setHeader("Sunset", dep.sunset);++ // RFC 8288 Link relations: "deprecation" (RFC 9745) for docs, and+ // "successor-version" (RFC 5829) for the replacement endpoint.+ const links: string[] = [];+ if (dep.docs) links.push(`<${dep.docs}>; rel="deprecation"`);+ if (dep.replacement) {+ links.push(`<${dep.replacement}>; rel="successor-version"`);+ }+ if (links.length) res.setHeader("Link", links.join(", "));+}+export function deprecationMiddleware(key: DeprecationKey) {const dep: Deprecation = DEPRECATIONS[key];return (req: Request, res: Response, next: NextFunction) => {- // RFC 9745 Deprecation header.- res.setHeader("Deprecation", dep.deprecatedAt);- // RFC 8594 Sunset header.- if (dep.sunset) res.setHeader("Sunset", dep.sunset);+ setDeprecationHeaders(res, dep);+ applyNotice(res, { message: dep.message, replacement: dep.replacement });+ next();+ };+}- // RFC 8288 Link relations: "deprecation" (RFC 9745) for docs, and- // "successor-version" (RFC 5829) for the replacement endpoint.- const links: string[] = [];- if (dep.docs) links.push(`<${dep.docs}>; rel="deprecation"`);- if (dep.replacement) {- links.push(`<${dep.replacement}>; rel="successor-version"`);- }- applyNotice(res, {- message: dep.message,- links,+// For an endpoint past its sunset. Mount it ahead of auth, so every caller gets+// the pointer and nothing is rate limited, credit checked or billed.+export function removedEndpointHandler(key: DeprecationKey) {+ const dep: Deprecation = DEPRECATIONS[key];+ return (req: Request, res: Response) => {+ const { origin, integration } = req.query;+ logger.warn("Removed endpoint called", {+ module: "deprecations",+ deprecation: key,+ path: req.baseUrl + req.path,+ origin: requestOrigin(+ { origin: typeof origin === "string" ? origin : undefined },+ req,+ ),+ integration:+ resolveIntegration(+ typeof integration === "string" ? integration : undefined,+ req,+ ) ?? null,+ userAgent: req.headers["user-agent"],+ });++ setDeprecationHeaders(res, dep);+ res.status(410).json({+ success: false,+ code: "ENDPOINT_REMOVED",+ error: dep.message,+ detail: dep.message,replacement: dep.replacement,+ documentation_url: dep.docs,});- next();};}diff --git a/apps/api/src/lib/research-keyless.ts b/apps/api/src/lib/research-keyless.tsindex 43a2ebd51..e1b18b59c 100644--- a/apps/api/src/lib/research-keyless.ts+++ b/apps/api/src/lib/research-keyless.ts@@ -6,10 +6,7 @@ function researchKeylessOperation(): ResearchKeylessOperation | null {const segments = req.path.toLowerCase().split("/").filter(Boolean);const papersIndex = segments.indexOf("papers");- // Paper paths win, so a paper id of "github" is not the github route.- if (papersIndex === -1) {- return segments.includes("github") ? "github" : null;- }+ if (papersIndex === -1) return null;const rest = segments.slice(papersIndex + 1);if (rest.length === 0) return "search";diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 4865edb9e..771d472db 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -60,7 +60,10 @@ import {import { queueStatusController } from "../controllers/v2/queue-status";import { creditUsageHistoricalController } from "../controllers/v2/credit-usage-historical";import { tokenUsageHistoricalController } from "../controllers/v2/token-usage-historical";-import { deprecationMiddleware } from "../lib/deprecations";+import {+ deprecationMiddleware,+ removedEndpointHandler,+} from "../lib/deprecations";import { isResearchKeylessDisabled } from "../lib/research-keyless";import { agentController } from "../controllers/v2/agent";import { agentStatusController } from "../controllers/v2/agent-status";@@ -733,6 +736,12 @@ v2Router.post();if (config.RESEARCH_PROXY_URL) {+ // Register before the /search/research and /research mounts so the 410 answers ahead of their auth.+ v2Router.get(+ ["/search/research/github", "/research/github"],+ removedEndpointHandler("v2_research_github_search"),+ );+v2Router.use("/search/research",authMiddleware(RateLimiterMode.Research, {diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 1c7eaea87..636392a4d 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -427,7 +427,6 @@ type LoggedRequest = {| "research_paper_inspect"| "research_paper_read"| "research_related_papers"- | "research_github_search"| "code_search"| "gov_search";api_version: string;@@ -1125,7 +1124,6 @@ export type ResearchRequestKind =| "research_paper_inspect"| "research_paper_read"| "research_related_papers"- | "research_github_search"| "code_search"| "gov_search";@@ -1134,7 +1132,6 @@ export type ResearchTableName =| "research_paper_inspects"| "research_paper_reads"| "research_related_papers"- | "research_github_searches"| "code_searches"| "gov_searches";