Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Disable keyless Research Index paper access by default
Claudec838e6c7cec5
diff --git a/apps/api/src/__tests__/routes/research-keyless.routes.test.ts b/apps/api/src/__tests__/routes/research-keyless.routes.test.tsindex 4c629da77..d5276f91c 100644--- a/apps/api/src/__tests__/routes/research-keyless.routes.test.ts+++ b/apps/api/src/__tests__/routes/research-keyless.routes.test.ts@@ -1,14 +1,6 @@import express from "express";import request from "supertest";-// Temporary Research Index keyless mitigation (RESEARCH_KEYLESS_DISABLED).-//-// The end-to-end coverage lives in snips/v2/research.test.ts, which needs a-// running API. These tests own the parts a live server can't vary per-request:-// how the env flag parses, how a request path maps onto a paper operation, and-// that the real auth middleware refuses a keyless call to a disabled operation-// with the "bring an API key" 401 rather than a 500 or a rate-limit error.-const KEYLESS_ENV = {KEYLESS_REQUESTS_PER_DAY: "100",KEYLESS_CREDITS_PER_DAY: "100",@@ -33,8 +25,6 @@ async function loadWithFlag(flag: string | undefined) {return { config, isResearchKeylessDisabled };}-// Mirrors the Research Index mount in routes/v2.ts, so the gate sees the same-// paths (mount prefix stripped) that production hands it.async function appWithResearchGate(flag: string | undefined) {const { isResearchKeylessDisabled } = await loadWithFlag(flag);const { authMiddleware } = await import("../../routes/shared.js");@@ -64,34 +54,25 @@ function fakeRequest(path: string, query: Record<string, unknown> = {}) {}describe("RESEARCH_KEYLESS_DISABLED parsing", () => {- it("leaves keyless untouched when unset", async () => {+ it("covers every paper operation by default when unset", async () => {const { config, isResearchKeylessDisabled } = await loadWithFlag(undefined);- expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([]);- expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001"))).toBe(- false,- );- });-- it("treats an empty value as unset, so clearing it is the revert", async () => {- const { config } = await loadWithFlag("");-- expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([]);- });-- it("applies the default scope when switched on, keeping paper search keyless", async () => {- const { config } = await loadWithFlag("true");-expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([+ "search","inspect","read","similar",]);- expect(config.RESEARCH_KEYLESS_DISABLED).not.toContain("search");+ expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001"))).toBe(+ true,+ );+ expect(+ isResearchKeylessDisabled(fakeRequest("/papers", { query: "rag" })),+ ).toBe(true);});- it("covers every paper operation with all", async () => {- const { config } = await loadWithFlag("all");+ it("treats an empty value as unset, so it also gets the default scope", async () => {+ const { config } = await loadWithFlag("");expect(config.RESEARCH_KEYLESS_DISABLED).toEqual(["search",@@ -101,6 +82,30 @@ describe("RESEARCH_KEYLESS_DISABLED parsing", () => {]);});+ it("restores keyless access when explicitly switched off", async () => {+ for (const flag of ["false", "none", "0", "off", "no"]) {+ const { config, isResearchKeylessDisabled } = await loadWithFlag(flag);++ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([]);+ expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001"))).toBe(+ false,+ );+ }+ });++ it("covers every paper operation when switched on explicitly", async () => {+ for (const flag of ["true", "all", "on", "1", "yes"]) {+ const { config } = await loadWithFlag(flag);++ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([+ "search",+ "inspect",+ "read",+ "similar",+ ]);+ }+ });+it("accepts an explicit operation list", async () => {const { config } = await loadWithFlag("inspect, similar");@@ -118,7 +123,6 @@ describe("Research Index paper operation gate", () => {"inspect,read,similar",);- // /papers/:id is inspect without a query and read with one.expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001"))).toBe(true,);@@ -130,11 +134,9 @@ describe("Research Index paper operation gate", () => {expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001/similar")),).toBe(true);- // Paper search is deliberately outside the default scope.expect(isResearchKeylessDisabled(fakeRequest("/papers", { query: "rag" })),).toBe(false);- // GitHub search is not a paper operation.expect(isResearchKeylessDisabled(fakeRequest("/github", { query: "x" })),).toBe(false);@@ -174,12 +176,10 @@ describe("Research Index paper operation gate", () => {});describe("keyless Research Index requests through the auth middleware", () => {- // routes/shared registers process-wide Prometheus metrics on import, so this- // file may only build the app once. It uses the default scope.let app: express.Express;beforeAll(async () => {- app = await appWithResearchGate("inspect,read,similar");+ app = await appWithResearchGate(undefined);});it("refuses a disabled operation with a 401 that asks for an API key", async () => {@@ -200,13 +200,22 @@ describe("keyless Research Index requests through the auth middleware", () => {}});- it("does not refuse keyless paper search under the default scope", async () => {+ it("refuses keyless paper search too, now that it is in the default scope", async () => {const response = await request(app).get("/v2/search/research/papers?query=transformers",);- // Whatever the keyless quota decides, search is never turned away for- // needing a key.+ expect(response.statusCode).toBe(401);+ expect(response.body.success).toBe(false);+ expect(response.body.error).toContain(+ "not supported by the keyless free tier",+ );+ expect(response.body.reachedController).toBeUndefined();+ });++ it("leaves the non-paper github operation keyless", async () => {+ const response = await request(app).get("/v2/search/research/github?q=x");+expect(response.body?.error ?? "").not.toContain("not supported by the keyless free tier",);diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.tsindex bed70a23e..0c6febe7a 100644--- a/apps/api/src/__tests__/snips/v2/research.test.ts+++ b/apps/api/src/__tests__/snips/v2/research.test.ts@@ -10,9 +10,6 @@ const KEYLESS_ENABLED =process.env.KEYLESS_REQUESTS_PER_DAY !== undefined &&process.env.KEYLESS_CREDITS_PER_DAY !== undefined;-// Temporary keyless mitigation. The harness passes the shell env to the server,-// so parsing RESEARCH_KEYLESS_DISABLED here (via the same config schema) tells-// us which paper operations the server refuses without an API key.const KEYLESS_DISABLED_OPERATIONS = config.RESEARCH_KEYLESS_DISABLED;const KEYLESS_SEARCH_ENABLED =KEYLESS_ENABLED && !KEYLESS_DISABLED_OPERATIONS.includes("search");@@ -218,8 +215,6 @@ describeIf(HAS_RESEARCH)("Research API", () => {}, 120000);});- // Win condition for the temporary RESEARCH_KEYLESS_DISABLED mitigation: the- // operations it covers require an API key, and keyed callers are untouched.describeIf(KEYLESS_INSPECT_DISABLED)("keyless research disabled", () => {it("refuses keyless paper inspect with a 401 telling the caller to use an API key", async () => {const res = await researchRaw(diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 339303db5..bf004c06d 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -16,9 +16,6 @@ const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>z.preprocess(value => (value === "" ? undefined : value), schema);-/* Research Index keyless mitigation */-// The paper operations served by the Research Index router. `/papers/:id` is-// two operations: `read` when a `query` is present, `inspect` when it isn't.const RESEARCH_PAPER_OPERATIONS = ["search","inspect",@@ -26,36 +23,23 @@ const RESEARCH_PAPER_OPERATIONS = ["similar",] as const;-// Scope used when the flag is switched on without naming operations. Keyless-// paper search stays open: that is where playground and pre-signup discovery-// traffic lives, so blocking it has real funnel cost, while keyless-// inspect/read/similar traffic is negligible.-const DEFAULT_RESEARCH_KEYLESS_DISABLED = [- "inspect",- "read",- "similar",-] as const;--// TEMPORARY (see lib/research-keyless.ts): reject keyless (unauthenticated)-// requests to Research Index paper operations. Accepts an on/off value or an-// explicit comma-separated scope so the blast radius can be widened or narrowed-// without a deploy. Unset/empty keeps today's behaviour, so unsetting it is the-// full revert.const researchKeylessDisabled = z.preprocess(value => {if (typeof value !== "string") return value;const raw = value.trim().toLowerCase();- if (["", "false", "0", "off", "no", "none"].includes(raw)) return [];- if (["true", "1", "on", "yes"].includes(raw)) {- return [...DEFAULT_RESEARCH_KEYLESS_DISABLED];+ if (raw === "") return undefined;+ if (["false", "0", "off", "no", "none"].includes(raw)) return [];+ if (["true", "1", "on", "yes", "all"].includes(raw)) {+ return [...RESEARCH_PAPER_OPERATIONS];}- if (raw === "all") return [...RESEARCH_PAPER_OPERATIONS];return raw.split(",").map(operation => operation.trim()).filter(Boolean);},- z.array(z.enum(RESEARCH_PAPER_OPERATIONS)).default([]),+ z+ .array(z.enum(RESEARCH_PAPER_OPERATIONS))+ .default([...RESEARCH_PAPER_OPERATIONS]),);const containsLoneSurrogate = (value: string): boolean => {@@ -96,12 +80,6 @@ const configSchema = z.object({z.string().trim().optional(),),RESEARCH_PROXY_URL: z.string().url().optional(),- // Temporary abuse mitigation for the Research Index paper endpoints: which- // paper operations refuse keyless (unauthenticated) callers. Unset or empty- // leaves keyless access exactly as it is today; "true" applies the default- // scope (inspect, read, similar); "all" covers every paper operation; a- // comma-separated list of search|inspect|read|similar sets the scope- // explicitly. API-key callers are never affected.RESEARCH_KEYLESS_DISABLED: researchKeylessDisabled,LABS_SEARCH_URL: z.string().url().optional(),LABS_SEARCH_SECRET: z.string().optional(),diff --git a/apps/api/src/lib/research-keyless.ts b/apps/api/src/lib/research-keyless.tsindex 9cfc0bdb8..df28bcdb4 100644--- a/apps/api/src/lib/research-keyless.ts+++ b/apps/api/src/lib/research-keyless.ts@@ -1,38 +1,15 @@import type { Request } from "express";import { config } from "../config";-/**- * TEMPORARY mitigation for keyless (unauthenticated) abuse of the Research- * Index paper endpoints. Keyless callers get a per-IP daily request/credit- * budget, but the paper operations are zero-credit and keyless requests never- * reach the per-minute limiter, so a harvest sharded across enough keyless- * identities enumerates the index for free. Until there is a permanent fix,- * `RESEARCH_KEYLESS_DISABLED` makes the selected paper operations require an- * API key. Unsetting the variable restores today's behaviour with no deploy.- *- * This only removes the keyless allowance: requests carrying an API key take- * the normal authenticated path and are unaffected, as are rate limits, credit- * costs, and billing.- */type ResearchPaperOperation = "search" | "inspect" | "read" | "similar";-/**- * The paper operation a Research Index request performs, or null if it is not a- * paper operation (e.g. `/github`). Matching starts at the `papers` segment, so- * this works both on the path Express hands a mounted router (mount prefix- * stripped) and on a full request path.- */function researchPaperOperation(req: Request): ResearchPaperOperation | null {- // Express routes case-insensitively by default, so classify that way too —- // otherwise `/PAPERS/2401.00001` would reach the controller ungated.const segments = req.path.toLowerCase().split("/").filter(Boolean);const papersIndex = segments.indexOf("papers");if (papersIndex === -1) return null;const rest = segments.slice(papersIndex + 1);if (rest.length === 0) return "search";- // `/papers/:id` serves both operations: reading a paper against a query, or- // inspecting its metadata when no query is given.if (rest.length === 1) {return req.query?.query === undefined ? "inspect" : "read";}@@ -40,10 +17,6 @@ function researchPaperOperation(req: Request): ResearchPaperOperation | null {return null;}-/**- * Whether this Research Index request must present an API key because its- * operation is in the `RESEARCH_KEYLESS_DISABLED` scope.- */export function isResearchKeylessDisabled(req: Request): boolean {const disabledOperations = config.RESEARCH_KEYLESS_DISABLED;if (disabledOperations.length === 0) return false;diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 4bf3ae5bc..3d8443b27 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -220,8 +220,6 @@ export function checkCreditsMiddleware(export function authMiddleware(rateLimiterMode: RateLimiterMode,options: {- // A predicate lets a router that mounts several operations behind one- // middleware allow keyless per request (see lib/research-keyless.ts).allowKeyless?: boolean | ((req: RequestWithMaybeAuth) => boolean);} = {},): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex e79942499..c45073164 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -651,8 +651,6 @@ v2Router.post();if (config.RESEARCH_PROXY_URL) {- // Keyless access here is temporarily revocable per paper operation via- // RESEARCH_KEYLESS_DISABLED; API-key callers are unaffected.v2Router.use("/search/research",authMiddleware(RateLimiterMode.Research, {