Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add a revocable flag that removes keyless access from Research Index papers

Claudebcd5e41f5356
diff --git a/apps/api/src/__tests__/routes/research-keyless.routes.test.ts b/apps/api/src/__tests__/routes/research-keyless.routes.test.ts
new file mode 100644
index 000000000..4c629da77
--- /dev/null
+++ b/apps/api/src/__tests__/routes/research-keyless.routes.test.ts
@@ -0,0 +1,214 @@
+import express from "express";
+import request from "supertest";
+
+// Temporary Research Index keyless mitigation (RESEARCH_KEYLESS_DISABLED).
+//
+// The end-to-end coverage lives in snips/v2/research.test.ts, which needs a
+// running API. These tests own the parts a live server can't vary per-request:
+// how the env flag parses, how a request path maps onto a paper operation, and
+// that the real auth middleware refuses a keyless call to a disabled operation
+// with the "bring an API key" 401 rather than a 500 or a rate-limit error.
+
+const KEYLESS_ENV = {
+ KEYLESS_REQUESTS_PER_DAY: "100",
+ KEYLESS_CREDITS_PER_DAY: "100",
+ USE_DB_AUTHENTICATION: "true",
+};
+
+async function loadWithFlag(flag: string | undefined) {
+ vi.resetModules();
+ for (const [key, value] of Object.entries(KEYLESS_ENV)) {
+ process.env[key] = value;
+ }
+ if (flag === undefined) {
+ delete process.env.RESEARCH_KEYLESS_DISABLED;
+ } else {
+ process.env.RESEARCH_KEYLESS_DISABLED = flag;
+ }
+
+ const { config } = await import("../../config.js");
+ const { isResearchKeylessDisabled } = await import(
+ "../../lib/research-keyless.js"
+ );
+ return { config, isResearchKeylessDisabled };
+}
+
+// Mirrors the Research Index mount in routes/v2.ts, so the gate sees the same
+// paths (mount prefix stripped) that production hands it.
+async function appWithResearchGate(flag: string | undefined) {
+ const { isResearchKeylessDisabled } = await loadWithFlag(flag);
+ const { authMiddleware } = await import("../../routes/shared.js");
+ const { RateLimiterMode } = await import("../../types.js");
+
+ const app = express();
+ const router = express.Router();
+ const reached = (_req: express.Request, res: express.Response) =>
+ res.status(200).json({ reachedController: true });
+ router.get("/papers", reached);
+ router.get("/papers/:id/similar", reached);
+ router.get("/papers/:id", reached);
+ router.get("/github", reached);
+
+ app.use(
+ "/v2/search/research",
+ authMiddleware(RateLimiterMode.Research, {
+ allowKeyless: req => !isResearchKeylessDisabled(req),
+ }),
+ router,
+ );
+ return app;
+}
+
+function fakeRequest(path: string, query: Record<string, unknown> = {}) {
+ return { path, query } as unknown as express.Request;
+}
+
+describe("RESEARCH_KEYLESS_DISABLED parsing", () => {
+ it("leaves keyless untouched when unset", async () => {
+ const { config, isResearchKeylessDisabled } = await loadWithFlag(undefined);
+
+ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([]);
+ expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001"))).toBe(
+ false,
+ );
+ });
+
+ it("treats an empty value as unset, so clearing it is the revert", async () => {
+ const { config } = await loadWithFlag("");
+
+ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([]);
+ });
+
+ it("applies the default scope when switched on, keeping paper search keyless", async () => {
+ const { config } = await loadWithFlag("true");
+
+ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([
+ "inspect",
+ "read",
+ "similar",
+ ]);
+ expect(config.RESEARCH_KEYLESS_DISABLED).not.toContain("search");
+ });
+
+ it("covers every paper operation with all", async () => {
+ const { config } = await loadWithFlag("all");
+
+ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual([
+ "search",
+ "inspect",
+ "read",
+ "similar",
+ ]);
+ });
+
+ it("accepts an explicit operation list", async () => {
+ const { config } = await loadWithFlag("inspect, similar");
+
+ expect(config.RESEARCH_KEYLESS_DISABLED).toEqual(["inspect", "similar"]);
+ });
+
+ it("refuses an unknown operation at boot instead of silently ignoring it", async () => {
+ await expect(loadWithFlag("inspect,papers")).rejects.toThrow();
+ });
+});
+
+describe("Research Index paper operation gate", () => {
+ it("maps each paper operation to its own flag entry", async () => {
+ const { isResearchKeylessDisabled } = await loadWithFlag(
+ "inspect,read,similar",
+ );
+
+ // /papers/:id is inspect without a query and read with one.
+ expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001"))).toBe(
+ true,
+ );
+ expect(
+ isResearchKeylessDisabled(
+ fakeRequest("/papers/2401.00001", { query: "attention" }),
+ ),
+ ).toBe(true);
+ expect(
+ isResearchKeylessDisabled(fakeRequest("/papers/2401.00001/similar")),
+ ).toBe(true);
+ // Paper search is deliberately outside the default scope.
+ expect(
+ isResearchKeylessDisabled(fakeRequest("/papers", { query: "rag" })),
+ ).toBe(false);
+ // GitHub search is not a paper operation.
+ expect(
+ isResearchKeylessDisabled(fakeRequest("/github", { query: "x" })),
+ ).toBe(false);
+ });
+
+ it("blocks only the named operation when the scope is narrowed", async () => {
+ const { isResearchKeylessDisabled } = await loadWithFlag("inspect");
+
+ expect(isResearchKeylessDisabled(fakeRequest("/papers/2401.00001"))).toBe(
+ true,
+ );
+ expect(
+ isResearchKeylessDisabled(
+ fakeRequest("/papers/2401.00001", { query: "attention" }),
+ ),
+ ).toBe(false);
+ expect(
+ isResearchKeylessDisabled(fakeRequest("/papers/2401.00001/similar")),
+ ).toBe(false);
+ });
+
+ it("classifies unstripped paths and casing the same way Express routes them", async () => {
+ const { isResearchKeylessDisabled } = await loadWithFlag("all");
+
+ expect(
+ isResearchKeylessDisabled(
+ fakeRequest("/v2/search/research/papers/2401.00001"),
+ ),
+ ).toBe(true);
+ expect(isResearchKeylessDisabled(fakeRequest("/PAPERS/2401.00001"))).toBe(
+ true,
+ );
+ expect(
+ isResearchKeylessDisabled(fakeRequest("/papers/2401.00001/SIMILAR")),
+ ).toBe(true);
+ });
+});
+
+describe("keyless Research Index requests through the auth middleware", () => {
+ // routes/shared registers process-wide Prometheus metrics on import, so this
+ // file may only build the app once. It uses the default scope.
+ let app: express.Express;
+
+ beforeAll(async () => {
+ app = await appWithResearchGate("inspect,read,similar");
+ });
+
+ it("refuses a disabled operation with a 401 that asks for an API key", async () => {
+ for (const path of [
+ "/v2/search/research/papers/2401.00001",
+ "/v2/search/research/papers/2401.00001?query=attention",
+ "/v2/search/research/papers/2401.00001/similar",
+ ]) {
+ const response = await request(app).get(path);
+
+ expect(response.statusCode).toBe(401);
+ expect(response.body.success).toBe(false);
+ expect(response.body.error).toContain(
+ "not supported by the keyless free tier",
+ );
+ expect(response.body.error).toContain("API key");
+ expect(response.body.reachedController).toBeUndefined();
+ }
+ });
+
+ it("does not refuse keyless paper search under the default scope", async () => {
+ const response = await request(app).get(
+ "/v2/search/research/papers?query=transformers",
+ );
+
+ // Whatever the keyless quota decides, search is never turned away for
+ // needing a key.
+ expect(response.body?.error ?? "").not.toContain(
+ "not supported by the keyless free tier",
+ );
+ });
+});
diff --git a/apps/api/src/__tests__/snips/v2/research.test.ts b/apps/api/src/__tests__/snips/v2/research.test.ts
index 95d0a6e0d..bed70a23e 100644
--- a/apps/api/src/__tests__/snips/v2/research.test.ts
+++ b/apps/api/src/__tests__/snips/v2/research.test.ts
@@ -10,6 +10,15 @@ const KEYLESS_ENABLED =
process.env.KEYLESS_REQUESTS_PER_DAY !== undefined &&
process.env.KEYLESS_CREDITS_PER_DAY !== undefined;
+// Temporary keyless mitigation. The harness passes the shell env to the server,
+// so parsing RESEARCH_KEYLESS_DISABLED here (via the same config schema) tells
+// us which paper operations the server refuses without an API key.
+const KEYLESS_DISABLED_OPERATIONS = config.RESEARCH_KEYLESS_DISABLED;
+const KEYLESS_SEARCH_ENABLED =
+ KEYLESS_ENABLED && !KEYLESS_DISABLED_OPERATIONS.includes("search");
+const KEYLESS_INSPECT_DISABLED =
+ KEYLESS_ENABLED && KEYLESS_DISABLED_OPERATIONS.includes("inspect");
+
const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
const sleepForBilling = () => sleep(40000);
@@ -161,7 +170,44 @@ describeIf(HAS_RESEARCH)("Research API", () => {
expect(res.body.success).toBe(false);
});
- describeIf(KEYLESS_ENABLED)("keyless research", () => {
+ it("serves every paper operation to an API key holder", async () => {
+ const identity = await idmux({
+ name: "research/keyed paper operations",
+ credits: 100,
+ });
+
+ const search = await researchRaw(
+ "/v2/search/research/papers",
+ { query: "attention is all you need", k: 1 },
+ identity,
+ );
+ expect(search.statusCode).toBe(200);
+ expect(search.body.success).toBe(true);
+
+ const inspect = await researchRaw(
+ "/v2/search/research/papers/1706.03762",
+ undefined,
+ identity,
+ );
+ expect(inspect.statusCode).toBe(200);
+
+ const read = await researchRaw(
+ "/v2/search/research/papers/1706.03762",
+ { query: "attention", k: 1 },
+ identity,
+ );
+ expect(read.statusCode).toBe(200);
+ expect(read.body.success).toBe(true);
+
+ const similar = await researchRaw(
+ "/v2/search/research/papers/1706.03762/similar",
+ { intent: "transformer architectures", k: 2 },
+ identity,
+ );
+ expect(similar.statusCode).toBe(200);
+ }, 120000);
+
+ describeIf(KEYLESS_SEARCH_ENABLED)("keyless research", () => {
it("permits keyless access on the canonical research index", async () => {
const res = await researchRaw("/v2/search/research/papers", {
query: "transformers",
@@ -172,6 +218,37 @@ describeIf(HAS_RESEARCH)("Research API", () => {
}, 120000);
});
+ // Win condition for the temporary RESEARCH_KEYLESS_DISABLED mitigation: the
+ // operations it covers require an API key, and keyed callers are untouched.
+ describeIf(KEYLESS_INSPECT_DISABLED)("keyless research disabled", () => {
+ it("refuses keyless paper inspect with a 401 telling the caller to use an API key", async () => {
+ const res = await researchRaw(
+ "/v2/search/research/papers/1706.03762",
+ undefined,
+ );
+
+ expect(res.statusCode).toBe(401);
+ expect(res.body.success).toBe(false);
+ expect(res.body.error).toContain("API key");
+ expect(res.body.results).toBeUndefined();
+ }, 120000);
+
+ it("keeps serving the same paper inspect to an API key holder", async () => {
+ const identity = await idmux({
+ name: "research/keyed inspect while keyless disabled",
+ credits: 100,
+ });
+
+ const res = await researchRaw(
+ "/v2/search/research/papers/1706.03762",
+ undefined,
+ identity,
+ );
+
+ expect(res.statusCode).toBe(200);
+ }, 120000);
+ });
+
describeIf(TEST_PRODUCTION)("research billing", () => {
it("research read-paper remains zero-credit", async () => {
const identity = await idmux({
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 1d8c894b6..339303db5 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -16,6 +16,48 @@ const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>
const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>
z.preprocess(value => (value === "" ? undefined : value), schema);
+/* Research Index keyless mitigation */
+// The paper operations served by the Research Index router. `/papers/:id` is
+// two operations: `read` when a `query` is present, `inspect` when it isn't.
+const RESEARCH_PAPER_OPERATIONS = [
+ "search",
+ "inspect",
+ "read",
+ "similar",
+] as const;
+
+// Scope used when the flag is switched on without naming operations. Keyless
+// paper search stays open: that is where playground and pre-signup discovery
+// traffic lives, so blocking it has real funnel cost, while keyless
+// inspect/read/similar traffic is negligible.
+const DEFAULT_RESEARCH_KEYLESS_DISABLED = [
+ "inspect",
+ "read",
+ "similar",
+] as const;
+
+// TEMPORARY (see lib/research-keyless.ts): reject keyless (unauthenticated)
+// requests to Research Index paper operations. Accepts an on/off value or an
+// explicit comma-separated scope so the blast radius can be widened or narrowed
+// without a deploy. Unset/empty keeps today's behaviour, so unsetting it is the
+// full revert.
+const researchKeylessDisabled = z.preprocess(
+ value => {
+ if (typeof value !== "string") return value;
+ const raw = value.trim().toLowerCase();
+ if (["", "false", "0", "off", "no", "none"].includes(raw)) return [];
+ if (["true", "1", "on", "yes"].includes(raw)) {
+ return [...DEFAULT_RESEARCH_KEYLESS_DISABLED];
+ }
+ if (raw === "all") return [...RESEARCH_PAPER_OPERATIONS];
+ return raw
+ .split(",")
+ .map(operation => operation.trim())
+ .filter(Boolean);
+ },
+ z.array(z.enum(RESEARCH_PAPER_OPERATIONS)).default([]),
+);
+
const containsLoneSurrogate = (value: string): boolean => {
for (let index = 0; index < value.length; index++) {
const codeUnit = value.charCodeAt(index);
@@ -54,6 +96,13 @@ const configSchema = z.object({
z.string().trim().optional(),
),
RESEARCH_PROXY_URL: z.string().url().optional(),
+ // Temporary abuse mitigation for the Research Index paper endpoints: which
+ // paper operations refuse keyless (unauthenticated) callers. Unset or empty
+ // leaves keyless access exactly as it is today; "true" applies the default
+ // scope (inspect, read, similar); "all" covers every paper operation; a
+ // comma-separated list of search|inspect|read|similar sets the scope
+ // explicitly. API-key callers are never affected.
+ RESEARCH_KEYLESS_DISABLED: researchKeylessDisabled,
LABS_SEARCH_URL: z.string().url().optional(),
LABS_SEARCH_SECRET: z.string().optional(),
diff --git a/apps/api/src/lib/research-keyless.ts b/apps/api/src/lib/research-keyless.ts
new file mode 100644
index 000000000..9cfc0bdb8
--- /dev/null
+++ b/apps/api/src/lib/research-keyless.ts
@@ -0,0 +1,53 @@
+import type { Request } from "express";
+import { config } from "../config";
+
+/**
+ * TEMPORARY mitigation for keyless (unauthenticated) abuse of the Research
+ * Index paper endpoints. Keyless callers get a per-IP daily request/credit
+ * budget, but the paper operations are zero-credit and keyless requests never
+ * reach the per-minute limiter, so a harvest sharded across enough keyless
+ * identities enumerates the index for free. Until there is a permanent fix,
+ * `RESEARCH_KEYLESS_DISABLED` makes the selected paper operations require an
+ * API key. Unsetting the variable restores today's behaviour with no deploy.
+ *
+ * This only removes the keyless allowance: requests carrying an API key take
+ * the normal authenticated path and are unaffected, as are rate limits, credit
+ * costs, and billing.
+ */
+type ResearchPaperOperation = "search" | "inspect" | "read" | "similar";
+
+/**
+ * The paper operation a Research Index request performs, or null if it is not a
+ * paper operation (e.g. `/github`). Matching starts at the `papers` segment, so
+ * this works both on the path Express hands a mounted router (mount prefix
+ * stripped) and on a full request path.
+ */
+function researchPaperOperation(req: Request): ResearchPaperOperation | null {
+ // Express routes case-insensitively by default, so classify that way too —
+ // otherwise `/PAPERS/2401.00001` would reach the controller ungated.
+ const segments = req.path.toLowerCase().split("/").filter(Boolean);
+ const papersIndex = segments.indexOf("papers");
+ if (papersIndex === -1) return null;
+
+ const rest = segments.slice(papersIndex + 1);
+ if (rest.length === 0) return "search";
+ // `/papers/:id` serves both operations: reading a paper against a query, or
+ // inspecting its metadata when no query is given.
+ if (rest.length === 1) {
+ return req.query?.query === undefined ? "inspect" : "read";
+ }
+ if (rest.length === 2 && rest[1] === "similar") return "similar";
+ return null;
+}
+
+/**
+ * Whether this Research Index request must present an API key because its
+ * operation is in the `RESEARCH_KEYLESS_DISABLED` scope.
+ */
+export function isResearchKeylessDisabled(req: Request): boolean {
+ const disabledOperations = config.RESEARCH_KEYLESS_DISABLED;
+ if (disabledOperations.length === 0) return false;
+
+ const operation = researchPaperOperation(req);
+ return operation !== null && disabledOperations.includes(operation);
+}
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 7e88dfd31..4bf3ae5bc 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -219,10 +219,19 @@ export function checkCreditsMiddleware(
export function authMiddleware(
rateLimiterMode: RateLimiterMode,
- options: { allowKeyless?: boolean } = {},
+ options: {
+ // A predicate lets a router that mounts several operations behind one
+ // middleware allow keyless per request (see lib/research-keyless.ts).
+ allowKeyless?: boolean | ((req: RequestWithMaybeAuth) => boolean);
+ } = {},
): (req: RequestWithMaybeAuth, res: Response, next: NextFunction) => void {
return (req, res, next) => {
(async () => {
+ const allowKeyless =
+ typeof options.allowKeyless === "function"
+ ? options.allowKeyless(req)
+ : options.allowKeyless;
+
let currentRateLimiterMode = rateLimiterMode;
if (
currentRateLimiterMode === RateLimiterMode.Extract &&
@@ -235,12 +244,10 @@ export function authMiddleware(
// currentRateLimiterMode = RateLimiterMode.ScrapeAgentPreview;
// }
- const auth = await authenticateUser(
- req,
- res,
- currentRateLimiterMode,
- options,
- );
+ const auth = await authenticateUser(req, res, currentRateLimiterMode, {
+ ...options,
+ allowKeyless,
+ });
if (!auth.success) {
if (!res.headersSent) {
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index b64c25621..e79942499 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -51,6 +51,7 @@ import { queueStatusController } from "../controllers/v2/queue-status";
import { creditUsageHistoricalController } from "../controllers/v2/credit-usage-historical";
import { tokenUsageHistoricalController } from "../controllers/v2/token-usage-historical";
import { deprecationMiddleware } from "../lib/deprecations";
+import { isResearchKeylessDisabled } from "../lib/research-keyless";
import { agentController } from "../controllers/v2/agent";
import { agentStatusController } from "../controllers/v2/agent-status";
import { agentCancelController } from "../controllers/v2/agent-cancel";
@@ -650,9 +651,13 @@ v2Router.post(
);
if (config.RESEARCH_PROXY_URL) {
+ // Keyless access here is temporarily revocable per paper operation via
+ // RESEARCH_KEYLESS_DISABLED; API-key callers are unaffected.
v2Router.use(
"/search/research",
- authMiddleware(RateLimiterMode.Research, { allowKeyless: true }),
+ authMiddleware(RateLimiterMode.Research, {
+ allowKeyless: req => !isResearchKeylessDisabled(req),
+ }),
createResearchRouter(),
);