Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): check credits before billed developer and GitHub searches (#5195)
Rakshith Ramprakash84404bf11901
diff --git a/apps/api/src/__tests__/routes/research-credit-gate.routes.test.ts b/apps/api/src/__tests__/routes/research-credit-gate.routes.test.tsnew file mode 100644index 000000000..141c644b4--- /dev/null+++ b/apps/api/src/__tests__/routes/research-credit-gate.routes.test.ts@@ -0,0 +1,121 @@+import express from "express";+import request from "supertest";+import {+ createDeveloperRouter,+ createResearchRouter,+} from "../../controllers/v2/research-proxy";+import { autumnService } from "../../services/autumn/autumn.service";++const upstreamCalls: string[] = [];++vi.mock("../../lib/research-upstream", () => ({+ fetchResearchUpstream: async (options: { path: string }) => {+ upstreamCalls.push(options.path);+ return {+ status: 200,+ ok: true,+ headers: new Headers(),+ text: async () => JSON.stringify({ success: true, results: [] }),+ };+ },+}));++vi.mock("../../services/logging/log_job", () => ({+ logRequest: async () => {},+ logResearchEndpoint: async () => {},+}));++vi.mock("../../lib/keyless", () => ({+ chargeKeylessCredits: async () => {},+}));++vi.mock("../../services/billing/credit_billing", () => ({+ billTeam: async () => {},+}));++vi.mock("../../services/autumn/autumn.service", () => ({+ autumnService: { checkCredits: vi.fn() },+ CREDITS_FEATURE_ID: "CREDITS",+ SEARCH_CREDITS_FEATURE_ID: "SEARCH_CREDITS",+}));++const checkCredits = vi.mocked(autumnService.checkCredits);++function appWith(orgId: string | undefined) {+ const app = express();+ app.use(express.json());+ app.use((req: any, _res, next) => {+ req.auth = { team_id: "team-test", org_id: orgId, plan: "standard" };+ req.acuc = { api_key_id: 4 };+ next();+ });+ app.use("/v2/search/research", createResearchRouter());+ app.use("/v2/search/developer", createDeveloperRouter({ root: true }));+ app.use("/v2/developer", createDeveloperRouter());+ return app;+}++const app = appWith("org-test");++beforeEach(() => {+ vi.clearAllMocks();+ upstreamCalls.length = 0;+});++describe.each([+ ["developer search", "/v2/search/developer?query=x"],+ ["legacy developer search", "/v2/developer/search?query=x"],+ ["GitHub research search", "/v2/search/research/github?query=x"],+])("%s credit gate", (_name, path) => {+ it("refuses with 402 when the team can't cover a search", async () => {+ checkCredits.mockResolvedValue({ allowed: false, remaining: 0 });++ const res = await request(app).get(path);++ expect(res.statusCode).toBe(402);+ expect(res.body.success).toBe(false);+ expect(upstreamCalls).toEqual([]);+ expect(checkCredits).toHaveBeenCalledWith(+ expect.objectContaining({+ teamId: "team-test",+ orgId: "org-test",+ value: 2,+ featureId: "SEARCH_CREDITS",+ }),+ );+ });++ it("serves the request when the team has credits", async () => {+ checkCredits.mockResolvedValue({ allowed: true, remaining: 500 });++ const res = await request(app).get(path);++ expect(res.statusCode).toBe(200);+ expect(upstreamCalls).toHaveLength(1);+ });++ it("fails open when billing can't answer", async () => {+ checkCredits.mockResolvedValue(null);++ const res = await request(app).get(path);++ expect(res.statusCode).toBe(200);+ expect(upstreamCalls).toHaveLength(1);+ });++ it("lets a caller with no org through, as other gates do", async () => {+ const res = await request(appWith(undefined)).get(path);++ expect(res.statusCode).toBe(200);+ expect(checkCredits).not.toHaveBeenCalled();+ });+});++it("leaves free paper search ungated", async () => {+ checkCredits.mockResolvedValue({ allowed: false, remaining: 0 });++ const res = await request(app).get("/v2/search/research/papers?query=x");++ expect(res.statusCode).toBe(200);+ expect(checkCredits).not.toHaveBeenCalled();+});diff --git a/apps/api/src/controllers/v2/__tests__/developer-access.test.ts b/apps/api/src/controllers/v2/__tests__/developer-access.test.tsindex 93f5a46ee..6efe258e0 100644--- a/apps/api/src/controllers/v2/__tests__/developer-access.test.ts+++ b/apps/api/src/controllers/v2/__tests__/developer-access.test.ts@@ -56,14 +56,14 @@ const KEYLESS_TEAM_ID = keylessTeamId(KEYLESS_IP);/** Lets the un-awaited controller (wrapped by `wrap`) and its `finally` run. */const flush = () => new Promise(resolve => setImmediate(resolve));-/** Pulls the GET handler out of the router's stack for either mount shape. */+/** Pulls the GET controller (after the credit gate) out of the router's stack for either mount shape. */function developerHandler(options: { root?: boolean } = {}) {const router: any = createDeveloperRouter(options);const path = options.root ? "/" : "/search";const layer = router.stack.find((l: any) => l.route?.path === path && l.route?.methods?.get,);- return layer.route.stack[0].handle;+ return layer.route.stack.at(-1).handle;}function makeReq(teamId: string, acuc: Record<string, unknown> | undefined) {diff --git a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.tsindex fcee47477..baca9d856 100644--- a/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts+++ b/apps/api/src/controllers/v2/__tests__/research-proxy-zdr.test.ts@@ -43,7 +43,7 @@ function routeHandler(router: any, path: string) {(candidate: any) =>candidate.route?.path === path && candidate.route?.methods?.get,);- return layer.route.stack[0].handle;+ return layer.route.stack.at(-1).handle;}function makeReq(diff --git a/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts b/apps/api/src/controllers/v2/__tests__/research-timeouts.test.tsindex 02709e7d5..b1ee34745 100644--- a/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts+++ b/apps/api/src/controllers/v2/__tests__/research-timeouts.test.ts@@ -45,7 +45,7 @@ function routeHandler(router: any, path: string) {(candidate: any) =>candidate.route?.path === path && candidate.route?.methods?.get,);- return layer.route.stack[0].handle;+ return layer.route.stack.at(-1).handle;}function makeReq(query: Record<string, unknown>, params = {}) {diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 8cc53948b..f91344477 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -23,7 +23,8 @@ import type {ResearchTableName,} from "../../services/logging/log_job";import type { RequestWithAuth } from "../v1/types";-import { wrap } from "../../routes/shared";+import { checkCreditsMiddleware, wrap } from "../../routes/shared";+import { SEARCH_CREDITS_FEATURE_ID } from "../../services/autumn/autumn.service";import { deprecationMiddleware } from "../../lib/deprecations";import {integrationFromHeader,@@ -32,6 +33,13 @@ import {import { requestOrigin } from "../../lib/request-origin";const SEARCH_CREDITS_PER_TEN_RESULTS = 2;+// Billed developer and GitHub searches are refused up front when the team+// can't cover the smallest search charge, the same gate /v2/search applies.+const searchCreditGate = () =>+ checkCreditsMiddleware(+ SEARCH_CREDITS_PER_TEN_RESULTS,+ SEARCH_CREDITS_FEATURE_ID,+ );const ZDR_SEARCH_CREDITS_PER_TEN_RESULTS = 10;const DEVELOPER_SEARCH_TIMEOUT_MS = 15_000;@@ -574,6 +582,7 @@ export function createResearchRouter(options: { legacy?: boolean } = {}) {router.get("/github",deprecationMiddleware("v2_research_github_search"),+ searchCreditGate(),wrap(createResearchController(githubSearchSchema,@@ -614,12 +623,13 @@ export function createDeveloperRouter(options: { root?: boolean } = {}) {),);+ const gate = searchCreditGate();if (options.root) {- router.get("/", controller);- router.post("/", controller);+ router.get("/", gate, controller);+ router.post("/", gate, controller);} else {- router.get("/search", controller);- router.post("/search", controller);+ router.get("/search", gate, controller);+ router.post("/search", gate, controller);}return router;