Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): proxy the fresh feed at /v2/fresh-vertical-feed/local/latest (#5744)
Karan Lokchandani065cc5fdf2ca
diff --git a/apps/api/src/__tests__/routes/fresh-feed.routes.test.ts b/apps/api/src/__tests__/routes/fresh-feed.routes.test.tsnew file mode 100644index 000000000..66b640889--- /dev/null+++ b/apps/api/src/__tests__/routes/fresh-feed.routes.test.ts@@ -0,0 +1,194 @@+import express from "express";+import http from "node:http";+import type { AddressInfo } from "node:net";+import request from "supertest";++const mocks = vi.hoisted(() => ({+ flags: {} as Record<string, unknown>,+ logRequest: vi.fn(),+}));++vi.mock("../../lib/logger", () => {+ const logger = {+ child: () => logger,+ error: vi.fn(),+ warn: vi.fn(),+ info: vi.fn(),+ debug: vi.fn(),+ };+ return { logger };+});++vi.mock("../../services/logging/log_job", () => ({+ logRequest: mocks.logRequest,+}));++vi.mock("../../routes/shared", () => ({ wrap: (fn: any) => fn }));++import { createFreshFeedRouter } from "../../routes/fresh-feed";++const PAGE =+ '{"feed":"local","events":[{"n":1.0,"t":"\\u00e9"}],"next_after":7,"more":false}';++const upstream = {+ requests: [] as { url?: string; authorization?: string }[],+ status: 200,+ body: PAGE,+ silent: false,+};++const upstreamServer = http.createServer((req, res) => {+ upstream.requests.push({+ url: req.url,+ authorization: req.headers.authorization,+ });+ if (upstream.silent) return;+ res.writeHead(upstream.status, { "content-type": "application/json" });+ res.end(upstream.body);+});++const app = express();+app.use((req: any, _res, next) => {+ req.auth = { team_id: "team-1" };+ req.acuc = { flags: mocks.flags };+ next();+});++const latest = (query = "") =>+ request(app)+ .get("/v2/fresh-vertical-feed/local/latest" + query)+ .set("Authorization", "Bearer fc-caller-key");++beforeAll(async () => {+ await new Promise<void>(resolve =>+ upstreamServer.listen(0, "127.0.0.1", resolve),+ );+ const { port } = upstreamServer.address() as AddressInfo;+ app.use(+ "/v2/fresh-vertical-feed",+ createFreshFeedRouter(`http://127.0.0.1:${port}/`),+ );+});++afterAll(async () => {+ upstreamServer.closeAllConnections();+ await new Promise(resolve => upstreamServer.close(resolve));+});++beforeEach(() => {+ vi.clearAllMocks();+ mocks.logRequest.mockResolvedValue(undefined);+ mocks.flags = { freshFeed: true };+ upstream.requests.length = 0;+ upstream.status = 200;+ upstream.body = PAGE;+ upstream.silent = false;+});++describe("GET /v2/fresh-vertical-feed/local/latest", () => {+ it("calls the feed with the caller's raw query and without the caller's key, and returns its body byte for byte", async () => {+ const query = "?limit=5&after=12&host=b+c&host=caf%C3%A9.example&x=";++ const response = await latest(query);++ expect(upstream.requests).toEqual([+ {+ url: "/fresh-vertical-feed/local/latest" + query,+ authorization: undefined,+ },+ ]);+ expect(response.status).toBe(200);+ expect(response.text).toBe(PAGE);+ });++ it("refuses a team without the flag before the feed is called", async () => {+ mocks.flags = {};++ const response = await latest("?after=0");++ expect(response.status).toBe(403);+ expect(response.body).toEqual({+ success: false,+ error: "This endpoint is not enabled for this team.",+ });+ expect(upstream.requests).toEqual([]);+ });++ it("returns the feed's 400 unchanged", async () => {+ upstream.status = 400;+ upstream.body = '{"error": "unknown parameter: colour"}';++ const response = await latest("?colour=red");++ expect(response.status).toBe(400);+ expect(response.text).toBe(upstream.body);+ });++ it.each([401, 500])(+ "answers 502 without the feed's body when the feed answers %i",+ async status => {+ upstream.status = status;+ upstream.body = '{"error":"upstream detail"}';++ const response = await latest();++ expect(response.status).toBe(502);+ expect(response.body).toEqual({+ success: false,+ error: "The request could not be completed.",+ });+ },+ );++ it("answers 504 when the feed does not answer in time", async () => {+ upstream.silent = true;+ const timeout = AbortSignal.timeout.bind(AbortSignal);+ const shortened = vi+ .spyOn(AbortSignal, "timeout")+ .mockImplementation(() => timeout(50));++ try {+ const response = await latest();++ expect(response.status).toBe(504);+ expect(response.body).toEqual({+ success: false,+ error: "The request timed out.",+ });+ } finally {+ shortened.mockRestore();+ }+ });++ it.each([+ [{ freshFeed: true }, false],+ [{ freshFeed: true, searchZDR: "forced-zdr" }, true],+ ])(+ "logs a fresh_feed request for flags %j with zeroDataRetention %s",+ async (flags, zeroDataRetention) => {+ mocks.flags = flags;++ await latest("?after=3");++ expect(mocks.logRequest).toHaveBeenCalledExactlyOnceWith(+ expect.objectContaining({+ kind: "fresh_feed",+ team_id: "team-1",+ zeroDataRetention,+ }),+ );+ },+ );++ it("compresses a large page for a caller that accepts gzip", async () => {+ upstream.body = JSON.stringify({+ feed: "local",+ events: [{ text: "<p>fresh</p>".repeat(500) }],+ });++ const response = await latest().set("Accept-Encoding", "gzip");++ expect(response.headers["content-encoding"]).toBe("gzip");+ expect(response.text).toBe(upstream.body);+ });+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex d660286da..5862f7bcb 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -144,6 +144,7 @@ const configSchema = z.object({GOV_INDEX_CALLS_PER_DAY_OVERRIDES: teamDailyLimits,LABS_SEARCH_URL: z.string().url().optional(),LABS_SEARCH_SECRET: z.string().optional(),+ FRESH_FEED_URL: emptyStringAsUndefined(z.string().url()),// ExpressEXPRESS_TRUST_PROXY: z.coerce.number().optional(),diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex a196161f6..dd8e71909 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1406,6 +1406,7 @@ export type TeamFlags = {researchBeta?: boolean;enrichBeta?: boolean;labsSearch?: boolean;+ freshFeed?: boolean;exchangeRetrieve?: boolean;professionalProfileCompanyDataBeta?: boolean;// The org's DPA (or partner amendment) restricts how its data may bediff --git a/apps/api/src/lib/__tests__/fresh-feed-config.test.ts b/apps/api/src/lib/__tests__/fresh-feed-config.test.tsnew file mode 100644index 000000000..2b977ae6c--- /dev/null+++ b/apps/api/src/lib/__tests__/fresh-feed-config.test.ts@@ -0,0 +1,15 @@+import { afterEach, expect, it, vi } from "vitest";++afterEach(() => {+ vi.unstubAllEnvs();+ vi.resetModules();+});++it("loads the config with an empty FRESH_FEED_URL and leaves it unset", async () => {+ vi.stubEnv("FRESH_FEED_URL", "");+ vi.resetModules();++ const { config } = await import("../../config.js");++ expect(config.FRESH_FEED_URL).toBeUndefined();+});diff --git a/apps/api/src/routes/fresh-feed.ts b/apps/api/src/routes/fresh-feed.tsnew file mode 100644index 000000000..c3f0fbc0c--- /dev/null+++ b/apps/api/src/routes/fresh-feed.ts@@ -0,0 +1,119 @@+import compression from "compression";+import express, { NextFunction, Request, Response } from "express";+import { v7 as uuidv7 } from "uuid";+import type { RequestWithAuth } from "../controllers/v1/types";+import { externalRequestId } from "../lib/external-request-id";+import { logger as rootLogger } from "../lib/logger";+import { requestOrigin } from "../lib/request-origin";+import { getEffectiveSearchForcedKind } from "../lib/safe-mode";+import { logRequest } from "../services/logging/log_job";+import { resolveIntegration } from "../utils/integration";+import { wrap } from "./shared";++const TIMEOUT_MS = 30_000;+const UPSTREAM_PATH = "/fresh-vertical-feed/local/latest";++function freshFeedError(res: Response, status: number, error: string) {+ return res.status(status).json({ success: false, error });+}++function requireFreshFeedFlag(req: Request, res: Response, next: NextFunction) {+ if (!(req as RequestWithAuth<any, any, any>).acuc?.flags?.freshFeed) {+ return freshFeedError(+ res,+ 403,+ "This endpoint is not enabled for this team.",+ );+ }+ next();+}++function eventCount(body: Buffer): number {+ try {+ const events = JSON.parse(body.toString()).events;+ return Array.isArray(events) ? events.length : 0;+ } catch {+ return 0;+ }+}++function freshFeedController(upstreamBase: string) {+ return async (req: Request, res: Response) => {+ const authedReq = req as RequestWithAuth<any, any, any>;+ const zeroDataRetention =+ getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !== null;++ const started = Date.now();+ const jobId = uuidv7();+ const logger = rootLogger.child({+ module: "api/fresh-feed",+ jobId,+ teamId: authedReq.auth.team_id,+ zeroDataRetention,+ });++ const queryStart = req.originalUrl.indexOf("?");+ const target =+ UPSTREAM_PATH ++ (queryStart === -1 ? "" : req.originalUrl.slice(queryStart));++ await logRequest({+ id: jobId,+ kind: "fresh_feed",+ api_version: "v2",+ external_request_id: externalRequestId(authedReq),+ team_id: authedReq.auth.team_id,+ origin: requestOrigin({}, req),+ integration: resolveIntegration(undefined, req) ?? null,+ target_hint: target,+ zeroDataRetention,+ api_key_id: authedReq.acuc?.api_key_id ?? null,+ });++ try {+ const upstream = await fetch(upstreamBase + target, {+ signal: AbortSignal.timeout(TIMEOUT_MS),+ });++ if (upstream.status !== 200 && upstream.status !== 400) {+ await upstream.body?.cancel();+ logger.error("Fresh feed upstream error", {+ status: upstream.status,+ seconds: (Date.now() - started) / 1000,+ });+ return freshFeedError(res, 502, "The request could not be completed.");+ }++ const body = Buffer.from(await upstream.arrayBuffer());+ const events = upstream.status === 200 ? eventCount(body) : 0;++ logger.info("Fresh feed request", {+ status: upstream.status,+ events,+ seconds: (Date.now() - started) / 1000,+ });++ const contentType = upstream.headers.get("content-type");+ if (contentType) res.setHeader("content-type", contentType);+ return res.status(upstream.status).send(body);+ } catch (error: unknown) {+ if (error instanceof DOMException && error.name === "TimeoutError") {+ logger.error("Fresh feed timed out");+ return freshFeedError(res, 504, "The request timed out.");+ }+ logger.error("Fresh feed proxy error", { error });+ return freshFeedError(res, 502, "The request could not be completed.");+ }+ };+}++export function createFreshFeedRouter(upstreamUrl: string) {+ const router = express.Router();+ router.get(+ "/local/latest",+ requireFreshFeedFlag,+ compression(),+ wrap(freshFeedController(upstreamUrl.replace(/\/+$/, ""))),+ );+ return router;+}diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.tsindex 5c1f827b9..85806ddaf 100644--- a/apps/api/src/routes/v2.ts+++ b/apps/api/src/routes/v2.ts@@ -3,6 +3,7 @@ import multer from "multer";import { config } from "../config";import { RateLimiterMode } from "../types";import { registerMcpActionLogReadRoute } from "./mcp-action-logs";+import { createFreshFeedRouter } from "./fresh-feed";import { SEARCH_CREDITS_FEATURE_ID } from "../services/autumn/autumn.service";import expressWs from "express-ws";import { searchController } from "../controllers/v2/search";@@ -777,3 +778,11 @@ if (config.SEARCH_PLATFORM_URL) {createGovRouter(),);}++if (config.FRESH_FEED_URL) {+ v2Router.use(+ "/fresh-vertical-feed",+ authMiddleware(RateLimiterMode.FreshFeed),+ createFreshFeedRouter(config.FRESH_FEED_URL),+ );+}diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 5f465c650..023e9a450 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -432,7 +432,8 @@ type LoggedRequest = {| "research_paper_read"| "research_related_papers"| "code_search"- | "gov_search";+ | "gov_search"+ | "fresh_feed";api_version: string;team_id: string;origin?: string;diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.tsindex 507ed5e9d..212f84d31 100644--- a/apps/api/src/services/rate-limiter.ts+++ b/apps/api/src/services/rate-limiter.ts@@ -42,6 +42,7 @@ const fallbackRateLimits: Record<RateLimiterMode, number> = {exchange: 100,// Catalogue reads only; no paid tool runs behind them.exchangeDiscover: 10000,+ freshFeed: 100,};/**diff --git a/apps/api/src/types.ts b/apps/api/src/types.tsindex a5a0d38fb..f45172bd8 100644--- a/apps/api/src/types.ts+++ b/apps/api/src/types.ts@@ -163,6 +163,7 @@ export enum RateLimiterMode {Labs = "labs",Exchange = "exchange",ExchangeDiscover = "exchangeDiscover",+ FreshFeed = "freshFeed",}export type AuthResponse =