Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add a revocable flag that removes keyless access from Research Index papers
Claudebcd5e41f5356
diff --git a/apps/api/src/__tests__/routes/research-keyless.routes.test.ts b/apps/api/src/__tests__/routes/research-keyless.routes.test.tsnew file mode 100644index 000000000..4c629da77--- /dev/null+++ b/apps/api/src/__tests__/routes/research-keyless.routes.test.ts@@ -0,0 +1,214 @@+import express from "express";+import request from "supertest";++// Temporary Research Index keyless mitigation (RESEARCH_KEYLESS_DISABLED).+//+// The end-to-end coverage lives in snips/v2/research.test.ts, which needs a+// running API. These tests own the parts a live server can't vary per-request:+// how the env flag parses, how a request path maps onto a paper operation, and+// that the real auth middleware refuses a keyless call to a disabled operation+// with the "bring an API key" 401 rather than a 500 or a rate-limit error.++const KEYLESS_ENV = {+ KEYLESS_REQUESTS_PER_DAY: "100",+ KEYLESS_CREDITS_PER_DAY: "100",+ USE_DB_AUTHENTICATION: "true",+};++async function loadWithFlag(flag: string | undefined) {+ vi.resetModules();+ for (const [key, value] of Object.entries(KEYLESS_ENV)) {+ process.env[key] = value;+ }+ if (flag === undefined) {+ delete process.env.RESEARCH_KEYLESS_DISABLED;+ } else {+ process.env.RESEARCH_KEYLESS_DISABLED = flag;+ }++ const { config } = await import("../../config.js");+ const { isResearchKeylessDisabled } = await import(+ "../../lib/research-keyless.js"+ );+ return { config, isResearchKeylessDisabled };+}++// Mirrors the Research Index mount in routes/v2.ts, so the gate sees the same+// paths (mount prefix stripped) that production hands it.+async function appWithResearchGate(flag: string | undefined) {+ const { isResearchKeylessDisabled } = await loadWithFlag(flag);+ const { authMiddleware } = await import("../../routes/shared.js");+ const { RateLimiterMode } = await import("../../types.js");++ const app = express();+ const router = express.Router();+ const reached = (_req: express.Request, res: express.Response) =>+ res.status(200).json({ reachedController: true });+ router.get("/papers", reached);+ router.get("/papers/:id/similar", reached);+ router.get("/papers/:id", reached);+ router.get("/github", reached);++ app.use(+ "/v2/search/research",+ authMiddleware(RateLimiterMode.Research, {+ allowKeyless: req => !isResearchKeylessDisabled(req),+ }),+ router,+ );+ return app;+}++function fakeRequest(path: string, query: Record<string, unknown> = {}) {+ return { path, query } as unknown as express.Request;+}++describe("RESEARCH_KEYLESS_DISABLED parsing", () => {+ it("leaves keyless untouched when unset", async () => {+ const { config, isResearchKeylessDisabled } = await loadWithFlag(undefined);++ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([]);+ expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001"))).toBe(+ false,+ );+ });++ it("treats an empty value as unset, so clearing it is the revert", async () => {+ const { config } = await loadWithFlag("");++ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([]);+ });++ it("applies the default scope when switched on, keeping paper search keyless", async () => {+ const { config } = await loadWithFlag("true");++ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([+ "inspect",+ "read",+ "similar",+ ]);+ expect(config.RESEARCH_KEYLESS_DISABLED).not.toContain("search");+ });++ it("covers every paper operation with all", async () => {+ const { config } = await loadWithFlag("all");++ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([+ "search",+ "inspect",+ "read",+ "similar",+ ]);+ });++ it("accepts an explicit operation list", async () => {+ const { config } = await loadWithFlag("inspect, similar");++ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual(["inspect", "similar"]);+ });++ it("refuses an unknown operation at boot instead of silently ignoring it", async () => {+ await expect(loadWithFlag("inspect,papers")).rejects.toThrow();+ });+});++describe("Research Index paper operation gate", () => {+ it("maps each paper operation to its own flag entry", async () => {+ const { isResearchKeylessDisabled } = await loadWithFlag(+ "inspect,read,similar",+ );++ // /papers/:id is inspect without a query and read with one.+ expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001"))).toBe(+ true,+ );+ expect(+ isResearchKeylessDisabled(+ fakeRequest("/papers/2401.00001", { query: "attention" }),+ ),+ ).toBe(true);+ expect(+ isResearchKeylessDisabled(fakeRequest("/papers/2401.00001/similar")),+ ).toBe(true);+ // Paper search is deliberately outside the default scope.+ expect(+ isResearchKeylessDisabled(fakeRequest("/papers", { query: "rag" })),+ ).toBe(false);+ // GitHub search is not a paper operation.+ expect(+ isResearchKeylessDisabled(fakeRequest("/github", { query: "x" })),+ ).toBe(false);+ });++ it("blocks only the named operation when the scope is narrowed", async () => {+ const { isResearchKeylessDisabled } = await loadWithFlag("inspect");++ expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001"))).toBe(+ true,+ );+ expect(+ isResearchKeylessDisabled(+ fakeRequest("/papers/2401.00001", { query: "attention" }),+ ),+ ).toBe(false);+ expect(+ isResearchKeylessDisabled(fakeRequest("/papers/2401.00001/similar")),+ ).toBe(false);+ });++ it("classifies unstripped paths and casing the same way Express routes them", async () => {+ const { isResearchKeylessDisabled } = await loadWithFlag("all");++ expect(+ isResearchKeylessDisabled(+ fakeRequest("/v2/search/research/papers/2401.00001"),+ ),+ ).toBe(true);+ expect(isResearchKeylessDisabled(fakeRequest("/PAPERS/2401.00001"))).toBe(+ true,+ );+ expect(+ isResearchKeylessDisabled(fakeRequest("/papers/2401.00001/SIMILAR")),+ ).toBe(true);+ });+});++describe("keyless Research Index requests through the auth middleware", () => {+ // routes/shared registers process-wide Prometheus metrics on import, so this+ // file may only build the app once. It uses the default scope.+ let app: express.Express;++ beforeAll(async () => {+ app = await appWithResearchGate("inspect,read,similar");+ });++ it("refuses a disabled operation with a 401 that asks for an API key", async () => {+ for (const path of [+ "/v2/search/research/papers/2401.00001",+ "/v2/search/research/papers/2401.00001?query=attention",+ "/v2/search/research/papers/2401.00001/similar",+ ]) {+ const response = await request(app).get(path);++ expect(response.statusCode).toBe(401);+ expect(response.body.success).toBe(false);+ expect(response.body.error).toContain(+ "not supported by the keyless free tier",+ );+ expect(response.body.error).toContain("API key");+ expect(response.body.reachedController).toBeUndefined();+ }+ });++ it("does not refuse keyless paper search under the default scope", async () => {+ const response = await request(app).get(+ "/v2/search/research/papers?query=transformers",+ );++ // Whatever the keyless quota decides, search is never turned away for+ // needing a key.+ expect(response.body?.error ?? "").not.toContain(+ "not supported by the keyless free tier",+ );+ });+});diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.tsindex 95d0a6e0d..bed70a23e 100644--- a/apps/api/src/__tests__/snips/v2/research.test.ts+++ b/apps/api/src/__tests__/snips/v2/research.test.ts@@ -10,6 +10,15 @@ const KEYLESS_ENABLED =process.env.KEYLESS_REQUESTS_PER_DAY !== undefined &&process.env.KEYLESS_CREDITS_PER_DAY !== undefined;+// Temporary keyless mitigation. The harness passes the shell env to the server,+// so parsing RESEARCH_KEYLESS_DISABLED here (via the same config schema) tells+// us which paper operations the server refuses without an API key.+const KEYLESS_DISABLED_OPERATIONS = config.RESEARCH_KEYLESS_DISABLED;+const KEYLESS_SEARCH_ENABLED =+ KEYLESS_ENABLED && !KEYLESS_DISABLED_OPERATIONS.includes("search");+const KEYLESS_INSPECT_DISABLED =+ KEYLESS_ENABLED && KEYLESS_DISABLED_OPERATIONS.includes("inspect");+const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));const sleepForBilling = () => sleep(40000);@@ -161,7 +170,44 @@ describeIf(HAS_RESEARCH)("Research API", () => {expect(res.body.success).toBe(false);});- describeIf(KEYLESS_ENABLED)("keyless research", () => {+ it("serves every paper operation to an API key holder", async () => {+ const identity = await idmux({+ name: "research/keyed paper operations",+ credits: 100,+ });++ const search = await researchRaw(+ "/v2/search/research/papers",+ { query: "attention is all you need", k: 1 },+ identity,+ );+ expect(search.statusCode).toBe(200);+ expect(search.body.success).toBe(true);++ const inspect = await researchRaw(+ "/v2/search/research/papers/1706.03762",+ undefined,+ identity,+ );+ expect(inspect.statusCode).toBe(200);++ const read = await researchRaw(+ "/v2/search/research/papers/1706.03762",+ { query: "attention", k: 1 },+ identity,+ );+ expect(read.statusCode).toBe(200);+ expect(read.body.success).toBe(true);++ const similar = await researchRaw(+ "/v2/search/research/papers/1706.03762/similar",+ { intent: "transformer architectures", k: 2 },+ identity,+ );+ expect(similar.statusCode).toBe(200);+ }, 120000);++ describeIf(KEYLESS_SEARCH_ENABLED)("keyless research", () => {it("permits keyless access on the canonical research index", async () => {const res = await researchRaw("/v2/search/research/papers", {query: "transformers",@@ -172,6 +218,37 @@ describeIf(HAS_RESEARCH)("Research API", () => {}, 120000);});+ // Win condition for the temporary RESEARCH_KEYLESS_DISABLED mitigation: the+ // operations it covers require an API key, and keyed callers are untouched.+ describeIf(KEYLESS_INSPECT_DISABLED)("keyless research disabled", () => {+ it("refuses keyless paper inspect with a 401 telling the caller to use an API key", async () => {+ const res = await researchRaw(+ "/v2/search/research/papers/1706.03762",+ undefined,+ );++ expect(res.statusCode).toBe(401);+ expect(res.body.success).toBe(false);+ expect(res.body.error).toContain("API key");+ expect(res.body.results).toBeUndefined();+ }, 120000);++ it("keeps serving the same paper inspect to an API key holder", async () => {+ const identity = await idmux({+ name: "research/keyed inspect while keyless disabled",+ credits: 100,+ });++ const res = await researchRaw(+ "/v2/search/research/papers/1706.03762",+ undefined,+ identity,+ );++ expect(res.statusCode).toBe(200);+ }, 120000);+ });+describeIf(TEST_PRODUCTION)("research billing", () => {it("research read-paper remains zero-credit", async () => {const identity = await idmux({diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 1d8c894b6..339303db5 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -16,6 +16,48 @@ const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>z.preprocess(value => (value === "" ? undefined : value), schema);+/* Research Index keyless mitigation */+// The paper operations served by the Research Index router. `/papers/:id` is+// two operations: `read` when a `query` is present, `inspect` when it isn't.+const RESEARCH_PAPER_OPERATIONS = [+ "search",+ "inspect",+ "read",+ "similar",+] as const;++// Scope used when the flag is switched on without naming operations. Keyless+// paper search stays open: that is where playground and pre-signup discovery+// traffic lives, so blocking it has real funnel cost, while keyless+// inspect/read/similar traffic is negligible.+const DEFAULT_RESEARCH_KEYLESS_DISABLED = [+ "inspect",+ "read",+ "similar",+] as const;++// TEMPORARY (see lib/research-keyless.ts): reject keyless (unauthenticated)+// requests to Research Index paper operations. Accepts an on/off value or an+// explicit comma-separated scope so the blast radius can be widened or narrowed+// without a deploy. Unset/empty keeps today's behaviour, so unsetting it is the+// full revert.+const researchKeylessDisabled = z.preprocess(+ value => {+ if (typeof value !== "string") return value;+ const raw = value.trim().toLowerCase();+ if (["", "false", "0", "off", "no", "none"].includes(raw)) return [];+ if (["true", "1", "on", "yes"].includes(raw)) {+ return [...DEFAULT_RESEARCH_KEYLESS_DISABLED];+ }+ if (raw === "all") return [...RESEARCH_PAPER_OPERATIONS];+ return raw+ .split(",")+ .map(operation => operation.trim())+ .filter(Boolean);+ },+ z.array(z.enum(RESEARCH_PAPER_OPERATIONS)).default([]),+);+const containsLoneSurrogate = (value: string): boolean => {for (let index = 0; index < value.length; index++) {const codeUnit = value.charCodeAt(index);@@ -54,6 +96,13 @@ const configSchema = z.object({z.string().trim().optional(),),RESEARCH_PROXY_URL: z.string().url().optional(),+ // Temporary abuse mitigation for the Research Index paper endpoints: which+ // paper operations refuse keyless (unauthenticated) callers. Unset or empty+ // leaves keyless access exactly as it is today; "true" applies the default+ // scope (inspect, read, similar); "all" covers every paper operation; a+ // comma-separated list of search|inspect|read|similar sets the scope+ // explicitly. API-key callers are never affected.+ RESEARCH_KEYLESS_DISABLED: researchKeylessDisabled,LABS_SEARCH_URL: z.string().url().optional(),LABS_SEARCH_SECRET: z.string().optional(),diff --git a/apps/api/src/lib/research-keyless.ts b/apps/api/src/lib/research-keyless.tsnew file mode 100644index 000000000..9cfc0bdb8--- /dev/null+++ b/apps/api/src/lib/research-keyless.ts@@ -0,0 +1,53 @@+import type { Request } from "express";+import { config } from "../config";++/**+ * TEMPORARY mitigation for keyless (unauthenticated) abuse of the Research+ * Index paper endpoints. Keyless callers get a per-IP daily request/credit+ * budget, but the paper operations are zero-credit and keyless requests never+ * reach the per-minute limiter, so a harvest sharded across enough keyless+ * identities enumerates the index for free. Until there is a permanent fix,+ * `RESEARCH_KEYLESS_DISABLED` makes the selected paper operations require an+ * API key. Unsetting the variable restores today's behaviour with no deploy.+ *+ * This only removes the keyless allowance: requests carrying an API key take+ * the normal authenticated path and are unaffected, as are rate limits, credit+ * costs, and billing.+ */+type ResearchPaperOperation = "search" | "inspect" | "read" | "similar";++/**+ * The paper operation a Research Index request performs, or null if it is not a+ * paper operation (e.g. `/github`). Matching starts at the `papers` segment, so+ * this works both on the path Express hands a mounted router (mount prefix+ * stripped) and on a full request path.+ */+function researchPaperOperation(req: Request): ResearchPaperOperation | null {+ // Express routes case-insensitively by default, so classify that way too —+ // otherwise `/PAPERS/2401.00001` would reach the controller ungated.+ const segments = req.path.toLowerCase().split("/").filter(Boolean);+ const papersIndex = segments.indexOf("papers");+ if (papersIndex === -1) return null;++ const rest = segments.slice(papersIndex + 1);+ if (rest.length === 0) return "search";+ // `/papers/:id` serves both operations: reading a paper against a query, or+ // inspecting its metadata when no query is given.+ if (rest.length === 1) {+ return req.query?.query === undefined ? "inspect" : "read";+ }+ if (rest.length === 2 && rest[1] === "similar") return "similar";+ return null;+}++/**+ * Whether this Research Index request must present an API key because its+ * operation is in the `RESEARCH_KEYLESS_DISABLED` scope.+ */+export function isResearchKeylessDisabled(req: Request): boolean {+ const disabledOperations = config.RESEARCH_KEYLESS_DISABLED;+ if (disabledOperations.length === 0) return false;++ const operation = researchPaperOperation(req);+ return operation !== null && disabledOperations.includes(operation);+}diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 7e88dfd31..4bf3ae5bc 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -219,10 +219,19 @@ export function checkCreditsMiddleware(export function authMiddleware(rateLimiterMode: RateLimiterMode,- options: { allowKeyless?: boolean } = {},+ options: {+ // A predicate lets a router that mounts several operations behind one+ // middleware allow keyless per request (see lib/research-keyless.ts).+ allowKeyless?: boolean | ((req: RequestWithMaybeAuth) => boolean);+ } = {},): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {return (req, res, next) => {(async () => {+ const allowKeyless =+ typeof options.allowKeyless === "function"+ ? options.allowKeyless(req)+ : options.allowKeyless;+let currentRateLimiterMode = rateLimiterMode;if (currentRateLimiterMode === RateLimiterMode.Extract &&@@ -235,12 +244,10 @@ export function authMiddleware(// currentRateLimiterMode = RateLimiterMode.ScrapeAgentPreview;// }- const auth = await authenticateUser(- req,- res,- currentRateLimiterMode,- options,- );+ const auth = await authenticateUser(req, res, currentRateLimiterMode, {+ ...options,+ allowKeyless,+ });if (!auth.success) {if (!res.headersSent) {diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex b64c25621..e79942499 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -51,6 +51,7 @@ import { queueStatusController } from "../controllers/v2/queue-status";import { creditUsageHistoricalController } from "../controllers/v2/credit-usage-historical";import { tokenUsageHistoricalController } from "../controllers/v2/token-usage-historical";import { deprecationMiddleware } from "../lib/deprecations";+import { isResearchKeylessDisabled } from "../lib/research-keyless";import { agentController } from "../controllers/v2/agent";import { agentStatusController } from "../controllers/v2/agent-status";import { agentCancelController } from "../controllers/v2/agent-cancel";@@ -650,9 +651,13 @@ v2Router.post();if (config.RESEARCH_PROXY_URL) {+ // Keyless access here is temporarily revocable per paper operation via+ // RESEARCH_KEYLESS_DISABLED; API-key callers are unaffected.v2Router.use("/search/research",- authMiddleware(RateLimiterMode.Research, { allowKeyless: true }),+ authMiddleware(RateLimiterMode.Research, {+ allowKeyless: req => !isResearchKeylessDisabled(req),+ }),createResearchRouter(),);