Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api)!: remove the developerBeta entitlement gate (#4228)
Kris Oei31141f069f46
diff --git a/apps/api/src/__tests__/snips/v2/developer.test.ts b/apps/api/src/__tests__/snips/v2/developer.test.tsindex 077a48e91..a07fe1e30 100644--- a/apps/api/src/__tests__/snips/v2/developer.test.ts+++ b/apps/api/src/__tests__/snips/v2/developer.test.ts@@ -10,9 +10,10 @@ const KEYLESS_ENABLED =process.env.KEYLESS_REQUESTS_PER_DAY !== undefined &&process.env.KEYLESS_CREDITS_PER_DAY !== undefined;-const CANONICAL_PATH = "/v2/developer/search";-const ALIAS_PATH = "/v2/search/developer";-const SERVING_PATHS = [CANONICAL_PATH, ALIAS_PATH];+const CANONICAL_PATH = "/v2/search/developer";+/** Compatibility mount kept for published CLI/MCP builds; removed once they ship on CANONICAL_PATH. */+const LEGACY_PATH = "/v2/developer/search";+const SERVING_PATHS = [CANONICAL_PATH, LEGACY_PATH];const COVERAGE_STATUSES = ["ok", "degraded", "unavailable", "skipped"];@@ -214,8 +215,8 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {}, 120000);describeIf(KEYLESS_ENABLED)("keyless developer search", () => {- it("permits keyless access on the alias mount", async () => {- const res = await researchRaw(ALIAS_PATH, {+ it("permits keyless access on the canonical mount", async () => {+ const res = await researchRaw(CANONICAL_PATH, {query: "retry backoff",k: 1,});@@ -225,8 +226,8 @@ describeIf(HAS_RESEARCH)("Developer Search API", () => {expect(Array.isArray(res.body.results)).toBe(true);}, 120000);- it("refuses keyless access on the canonical mount", async () => {- const res = await researchRaw(CANONICAL_PATH, {+ it("refuses keyless access on the legacy mount", async () => {+ const res = await researchRaw(LEGACY_PATH, {query: "retry backoff",k: 1,});diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex 23fc0bd9e..b27eccb10 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1313,7 +1313,6 @@ export type TeamFlags = {searchFeedbackOptOut?: boolean;researchBeta?: boolean;enrichBeta?: boolean;- developerBeta?: boolean;labsSearch?: boolean;professionalProfileCompanyDataBeta?: boolean;organizationDataSourceAccess?: Record<diff --git a/apps/api/src/controllers/v2/__tests__/developer-access.test.ts b/apps/api/src/controllers/v2/__tests__/developer-access.test.tsnew file mode 100644index 000000000..7841be720--- /dev/null+++ b/apps/api/src/controllers/v2/__tests__/developer-access.test.ts@@ -0,0 +1,179 @@+import { vi } from "vitest";++const mocks = vi.hoisted(() => ({+ logRequest: vi.fn(),+ logSearch: vi.fn(),+ logResearchEndpoint: vi.fn(),+ fetchResearchUpstream: vi.fn(),+ chargeKeylessCredits: vi.fn().mockResolvedValue(undefined),+}));++vi.mock("../../../services/logging/log_job", () => ({+ logRequest: mocks.logRequest,+ logSearch: mocks.logSearch,+ logResearchEndpoint: mocks.logResearchEndpoint,+}));++vi.mock("../../../lib/research-upstream", () => ({+ fetchResearchUpstream: mocks.fetchResearchUpstream,+}));++vi.mock("../../../services/billing/credit_billing", () => ({+ billTeam: vi.fn().mockResolvedValue(undefined),+}));++// Real `keylessTeamId` is kept so the test can build a production-shaped+// `preview_keyless_<ip>` team id (see ../../../lib/keyless.ts:44-46). Only+// `chargeKeylessCredits` β the function research-proxy.ts calls to actually+// charge a keyless caller's per-IP credit budget β is mocked, so this test+// stays a unit test of research-proxy.ts's wiring rather than exercising real+// Redis.+vi.mock("../../../lib/keyless", async importOriginal => {+ const actual = await importOriginal<typeof import("../../../lib/keyless")>();+ return { ...actual, chargeKeylessCredits: mocks.chargeKeylessCredits };+});++vi.mock("../../../lib/logger", () => ({+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ child: vi.fn().mockReturnThis(),+ },+}));++import { createDeveloperRouter } from "../research-proxy";+import { billTeam } from "../../../services/billing/credit_billing";+import { chargeKeylessCredits, keylessTeamId } from "../../../lib/keyless";++const TEAM_ID = "11111111-1111-1111-1111-111111111111";+// RFC 5737 TEST-NET-3 address: a valid IPv4 that's guaranteed non-routable,+// used only as a stand-in client IP for building a keyless team id.+const KEYLESS_IP = "203.0.113.7";+const KEYLESS_TEAM_ID = keylessTeamId(KEYLESS_IP);++/** Lets the un-awaited controller (wrapped by `wrap`) and its `finally` run. */+const flush = () => new Promise(resolve => setImmediate(resolve));++/** Pulls the GET handler out of the router's stack for either mount shape. */+function developerHandler(options: { root?: boolean } = {}) {+ const router: any = createDeveloperRouter(options);+ const path = options.root ? "/" : "/search";+ const layer = router.stack.find(+ (l: any) => l.route?.path === path && l.route?.methods?.get,+ );+ return layer.route.stack[0].handle;+}++function makeReq(teamId: string, acuc: Record<string, unknown> | undefined) {+ return {+ method: "GET",+ query: { query: "http client" },+ body: {},+ headers: {},+ auth: { team_id: teamId },+ acuc,+ } as any;+}++function makeAuthedReq(flags: Record<string, unknown>) {+ return makeReq(TEAM_ID, { api_key_id: 7, flags });+}++/**+ * A real keyless request: team id shaped like `chargeKeylessCredits`/+ * `billTeam` (via autumn's `isPreviewTeam`) key their behavior on β see+ * ../../../lib/keyless.ts:39-46 β and an `acuc` matching the object+ * production keyless callers actually get, `mockPreviewACUC(team_id, false)`+ * in ../../auth.ts:81-90 (assigned at ../../auth.ts:543), not "no acuc at+ * all".+ */+function makeKeylessReq() {+ return makeReq(KEYLESS_TEAM_ID, {+ api_key: "preview",+ api_key_id: 0,+ team_id: KEYLESS_TEAM_ID,+ org_id: "preview",+ flags: null,+ is_banned: false,+ is_extract: false,+ });+}++function makeRes() {+ const res: any = {+ status: vi.fn(),+ json: vi.fn(),+ send: vi.fn(),+ end: vi.fn(),+ setHeader: vi.fn(),+ };+ res.status.mockReturnValue(res);+ res.json.mockReturnValue(res);+ res.send.mockReturnValue(res);+ return res;+}++beforeEach(() => {+ vi.clearAllMocks();+ mocks.logRequest.mockResolvedValue(undefined);+ mocks.logResearchEndpoint.mockResolvedValue(undefined);+ // The controller consumes this as a `fetch` Response: `.ok`, `.status`,+ // `.headers.get()` and `await .text()`. A plain body object makes+ // `.headers.get()` throw and the handler answer 502 from its catch block,+ // which would let these tests pass without ever serving a real response.+ // At least one result is required for the request to bill any credits.+ mocks.fetchResearchUpstream.mockResolvedValue({+ ok: true,+ status: 200,+ headers: new Headers(),+ text: async () =>+ JSON.stringify({ success: true, results: [{ id: "repo-one" }] }),+ });+});++describe.each([+ { name: "/v2/search/developer", options: { root: true } },+ { name: "/v2/developer/search (compatibility mount)", options: {} },+])("developer endpoint access on $name", ({ options }) => {+ it("serves a team with no flags at all", async () => {+ const res = makeRes();+ await developerHandler(options)(makeAuthedReq({}), res);+ await flush();++ expect(res.status).not.toHaveBeenCalledWith(403);+ expect(mocks.fetchResearchUpstream).toHaveBeenCalled();+ expect(res.status).toHaveBeenCalledWith(200);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({ success: true }),+ );+ // Ordinary team billing runs for the real (non-keyless) team id.+ expect(billTeam).toHaveBeenCalledWith(+ TEAM_ID,+ expect.any(Number),+ expect.anything(),+ expect.anything(),+ );+ });++ it("serves a keyless caller (preview_keyless_<ip> team id)", async () => {+ const res = makeRes();+ await developerHandler(options)(makeKeylessReq(), res);+ await flush();++ expect(res.status).not.toHaveBeenCalledWith(403);+ expect(mocks.fetchResearchUpstream).toHaveBeenCalled();+ expect(res.status).toHaveBeenCalledWith(200);+ expect(res.json).toHaveBeenCalledWith(+ expect.objectContaining({ success: true }),+ );+ // Keyless credit charging runs for the keyless-shaped team id β this is+ // the assertion that regresses if the developer endpoint's keyless+ // billing path breaks (see research-proxy.ts:317-335).+ expect(chargeKeylessCredits).toHaveBeenCalledWith(+ KEYLESS_TEAM_ID,+ expect.any(Number),+ );+ });+});diff --git a/apps/api/src/controllers/v2/__tests__/developer-beta-gate.test.ts b/apps/api/src/controllers/v2/__tests__/developer-beta-gate.test.tsdeleted file mode 100644index 1e98e5e75..000000000--- a/apps/api/src/controllers/v2/__tests__/developer-beta-gate.test.ts+++ /dev/null@@ -1,100 +0,0 @@-import { vi } from "vitest";--const mocks = vi.hoisted(() => ({- logRequest: vi.fn(),- logSearch: vi.fn(),- logResearchEndpoint: vi.fn(),- fetchResearchUpstream: vi.fn(),-}));--vi.mock("../../../services/logging/log_job", () => ({- logRequest: mocks.logRequest,- logSearch: mocks.logSearch,- logResearchEndpoint: mocks.logResearchEndpoint,-}));--vi.mock("../../../lib/research-upstream", () => ({- fetchResearchUpstream: mocks.fetchResearchUpstream,-}));--vi.mock("../../../services/billing/credit_billing", () => ({- billTeam: vi.fn().mockResolvedValue(undefined),-}));--vi.mock("../../../lib/logger", () => ({- logger: {- info: vi.fn(),- warn: vi.fn(),- error: vi.fn(),- debug: vi.fn(),- child: vi.fn().mockReturnThis(),- },-}));--import { createDeveloperRouter } from "../research-proxy";--const TEAM_ID = "11111111-1111-1111-1111-111111111111";--/** Pulls the GET handler out of the router's stack for either mount shape. */-function developerHandler(options: { root?: boolean } = {}) {- const router: any = createDeveloperRouter(options);- const path = options.root ? "/" : "/search";- const layer = router.stack.find(- (l: any) => l.route?.path === path && l.route?.methods?.get,- );- return layer.route.stack[0].handle;-}--function makeReq(flags: Record<string, unknown> | null) {- return {- method: "GET",- query: { query: "http client" },- body: {},- headers: {},- auth: { team_id: TEAM_ID },- acuc: flags === null ? undefined : { api_key_id: 7, flags },- } as any;-}--function makeRes() {- const res: any = { status: vi.fn(), json: vi.fn(), end: vi.fn() };- res.status.mockReturnValue(res);- res.json.mockReturnValue(res);- return res;-}--beforeEach(() => {- vi.clearAllMocks();- mocks.logRequest.mockResolvedValue(undefined);- mocks.logResearchEndpoint.mockResolvedValue(undefined);-});--describe.each([- { name: "/v2/developer/search", options: {} },- { name: "/v2/search/developer (root alias)", options: { root: true } },-])("developer endpoint beta gate on $name", ({ options }) => {- it("403s a team without developerBeta and never calls upstream", async () => {- const res = makeRes();- await developerHandler(options)(makeReq({ developerBeta: false }), res);-- expect(res.status).toHaveBeenCalledWith(403);- expect(mocks.fetchResearchUpstream).not.toHaveBeenCalled();- expect(mocks.logRequest).not.toHaveBeenCalled();- });-- it("403s a keyless caller (no acuc at all)", async () => {- const res = makeRes();- await developerHandler(options)(makeReq(null), res);-- expect(res.status).toHaveBeenCalledWith(403);- expect(mocks.fetchResearchUpstream).not.toHaveBeenCalled();- });-- it("does not 403 a team with developerBeta", async () => {- mocks.fetchResearchUpstream.mockResolvedValue(null); // 404 path, past the gate- const res = makeRes();- await developerHandler(options)(makeReq({ developerBeta: true }), res);-- expect(res.status).not.toHaveBeenCalledWith(403);- });-});diff --git a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.tsindex fa61ec6f5..76586b036 100644--- a/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts+++ b/apps/api/src/controllers/v2/__tests__/search-developer-ledger.test.ts@@ -104,7 +104,7 @@ function makeReq(body: Record<string, any>, headers: Record<string, any> = {}) {body,headers,auth: { team_id: TEAM_ID },- acuc: { api_key_id: 7, flags: { developerBeta: true } },+ acuc: { api_key_id: 7, flags: {} },} as any;}@@ -299,20 +299,22 @@ describe("developer category code_searches ledger", () => {expect(body.creditsUsed).toBe(2);});- it("drops the developer category for a team without developerBeta", async () => {+ it("keeps the developer category for a team with no flags", async () => {mockExecuteSearch.mockResolvedValue(executeResult({ response: { web: [] } }));const req = makeReq({ query: "http client", categories: ["developer"] });- req.acuc = { api_key_id: 7, flags: null }; // unentitled, and keyless-equivalent+ req.acuc = { api_key_id: 7, flags: null }; // keyless-equivalent: no org flagsconst res = makeRes();await searchController(req, res);await flushAsync();- // Category stripped before execution -> no developer arm, no ledger row,- // and the search still succeeds (no error status).+ // The category survives to execution β no entitlement check any more.+ // (searchRequestSchema normalizes string categories to { type } objects+ // before they reach executeSearch, same as every other test in this file.)expect(mockExecuteSearch).toHaveBeenCalled();- expect(mockExecuteSearch.mock.calls[0][0].categories).toEqual([]);- expect(mockLogResearchEndpoint).not.toHaveBeenCalled();+ expect(mockExecuteSearch.mock.calls[0][0].categories).toEqual([+ { type: "developer" },+ ]);expect(res.status).not.toHaveBeenCalledWith(403);});});diff --git a/apps/api/src/controllers/v2/research-proxy.ts b/apps/api/src/controllers/v2/research-proxy.tsindex 6b826f2dc..b2539b4ab 100644--- a/apps/api/src/controllers/v2/research-proxy.ts+++ b/apps/api/src/controllers/v2/research-proxy.ts@@ -15,7 +15,6 @@ import type {ResearchTableName,} from "../../services/logging/log_job";import type { RequestWithAuth } from "../v1/types";-import type { TeamFlags } from "./types";import { wrap } from "../../routes/shared";import { integrationSchema } from "../../utils/integration";import { requestOrigin } from "../../lib/request-origin";@@ -143,12 +142,6 @@ type ResearchEndpointConfig = {req: RequestWithAuth<any, any, any>,) => string;billAs: "scrape" | "search";- /**- * Org flag that must be exactly `true` for this endpoint. Absent means- * ungated (the research endpoints). Keyless callers have no acuc and so no- * flags, which makes them unentitled by construction.- */- betaGate?: { flag: keyof NonNullable<TeamFlags>; error: string };};type ResearchController = (req: Request, res: Response) => Promise<any>;@@ -255,14 +248,6 @@ function createResearchController(return async (req, res: Response) => {const authedReq = req as RequestWithAuth<any, any, any>;- // Beta gate: fail closed before any logging, billing, or upstream call.- if (- endpoint.betaGate &&- authedReq.acuc?.flags?.[endpoint.betaGate.flag] !== true- ) {- return researchError(res, 403, endpoint.betaGate.error);- }-const started = Date.now();const jobId = uuidv7();const logger = rootLogger.child({@@ -512,11 +497,6 @@ export function createDeveloperRouter(options: { root?: boolean } = {}) {targetHint: params => String(params.query),upstreamPath: () => "/v2/code/search",billAs: "search",- betaGate: {- flag: "developerBeta",- error:- "Developer search is in beta and is not enabled for this team.",- },},),);diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 3f1d6c5ac..63e08aef2 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -76,28 +76,6 @@ export async function searchController(typeof req.body?.origin === "string" ? req.body.origin : undefined;req.body = searchRequestSchema.parse(req.body);- // Beta gate: the developer category is limited to teams with the- // developerBeta flag. Fail closed and silent β an unentitled team gets- // normal web results with the category dropped, no error. Keyless callers- // have no org and so no flags, which makes them unentitled. Runs before- // the key-restriction check below so an unentitled team is never told its- // key lacks access to a category that is about to be removed anyway.- if (wantsDeveloperCategory(req.body.categories as CategoryOption[])) {- if (req.acuc?.flags?.developerBeta !== true) {- // Expected, high-volume path (keyless + unentitled teams) β left- // unlogged on purpose to avoid log spam. See PR discussion.- // filter() widens the union that categories is declared as (either an- // all-string or an all-object array), so the result is cast back to- // assign it.- req.body.categories = (req.body.categories as CategoryOption[]).filter(- category =>- typeof category === "string"- ? category !== "developer"- : category.type !== "developer",- ) as typeof req.body.categories;- }- }-const requestedFormats = formatTypesOf(req.body.scrapeOptions?.formats);const keyRestriction = await checkKeyFormatRestriction(requestedFormats,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 466fb2c2a..d6b2075fd 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1574,7 +1574,6 @@ export type TeamFlags = {researchBeta?: boolean;menuBeta?: boolean;enrichBeta?: boolean;- developerBeta?: boolean;professionalProfileCompanyDataBeta?: boolean;organizationDataSourceAccess?: Record<string,diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex e0815c994..26904c8fa 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -646,12 +646,15 @@ if (config.RESEARCH_PROXY_URL) {createResearchRouter({ legacy: true }),);+ // Canonical: developer search is a subset of search, so it lives under it.v2Router.use("/search/developer",authMiddleware(RateLimiterMode.DeveloperSearch, { allowKeyless: true }),createDeveloperRouter({ root: true }),);+ // Compatibility only: the pre-GA path. Published CLI and MCP builds still+ // call it. Delete once those ship on /search/developer. Not documented.v2Router.use("/developer",authMiddleware(RateLimiterMode.DeveloperSearch),