Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): proxy the fresh feed at /v2/fresh-vertical-feed/local/latest (#5744)

Karan Lokchandani065cc5fdf2ca
diff --git a/apps/api/src/__tests__/routes/fresh-feed.routes.test.ts b/apps/api/src/__tests__/routes/fresh-feed.routes.test.ts
new file mode 100644
index 000000000..66b640889
--- /dev/null
+++ b/apps/api/src/__tests__/routes/fresh-feed.routes.test.ts
@@ -0,0 +1,194 @@
+import express from "express";
+import http from "node:http";
+import type { AddressInfo } from "node:net";
+import request from "supertest";
+
+const mocks = vi.hoisted(() => ({
+ flags: {} as Record<string, unknown>,
+ logRequest: vi.fn(),
+}));
+
+vi.mock("../../lib/logger", () => {
+ const logger = {
+ child: () => logger,
+ error: vi.fn(),
+ warn: vi.fn(),
+ info: vi.fn(),
+ debug: vi.fn(),
+ };
+ return { logger };
+});
+
+vi.mock("../../services/logging/log_job", () => ({
+ logRequest: mocks.logRequest,
+}));
+
+vi.mock("../../routes/shared", () => ({ wrap: (fn: any) => fn }));
+
+import { createFreshFeedRouter } from "../../routes/fresh-feed";
+
+const PAGE =
+ '{"feed":"local","events":[{"n":1.0,"t":"\\u00e9"}],"next_after":7,"more":false}';
+
+const upstream = {
+ requests: [] as { url?: string; authorization?: string }[],
+ status: 200,
+ body: PAGE,
+ silent: false,
+};
+
+const upstreamServer = http.createServer((req, res) => {
+ upstream.requests.push({
+ url: req.url,
+ authorization: req.headers.authorization,
+ });
+ if (upstream.silent) return;
+ res.writeHead(upstream.status, { "content-type": "application/json" });
+ res.end(upstream.body);
+});
+
+const app = express();
+app.use((req: any, _res, next) => {
+ req.auth = { team_id: "team-1" };
+ req.acuc = { flags: mocks.flags };
+ next();
+});
+
+const latest = (query = "") =>
+ request(app)
+ .get("/v2/fresh-vertical-feed/local/latest" + query)
+ .set("Authorization", "Bearer fc-caller-key");
+
+beforeAll(async () => {
+ await new Promise<void>(resolve =>
+ upstreamServer.listen(0, "127.0.0.1", resolve),
+ );
+ const { port } = upstreamServer.address() as AddressInfo;
+ app.use(
+ "/v2/fresh-vertical-feed",
+ createFreshFeedRouter(`http://127.0.0.1:${port}/`),
+ );
+});
+
+afterAll(async () => {
+ upstreamServer.closeAllConnections();
+ await new Promise(resolve => upstreamServer.close(resolve));
+});
+
+beforeEach(() => {
+ vi.clearAllMocks();
+ mocks.logRequest.mockResolvedValue(undefined);
+ mocks.flags = { freshFeed: true };
+ upstream.requests.length = 0;
+ upstream.status = 200;
+ upstream.body = PAGE;
+ upstream.silent = false;
+});
+
+describe("GET /v2/fresh-vertical-feed/local/latest", () => {
+ it("calls the feed with the caller's raw query and without the caller's key, and returns its body byte for byte", async () => {
+ const query = "?limit=5&after=12&host=b+c&host=caf%C3%A9.example&x=";
+
+ const response = await latest(query);
+
+ expect(upstream.requests).toEqual([
+ {
+ url: "/fresh-vertical-feed/local/latest" + query,
+ authorization: undefined,
+ },
+ ]);
+ expect(response.status).toBe(200);
+ expect(response.text).toBe(PAGE);
+ });
+
+ it("refuses a team without the flag before the feed is called", async () => {
+ mocks.flags = {};
+
+ const response = await latest("?after=0");
+
+ expect(response.status).toBe(403);
+ expect(response.body).toEqual({
+ success: false,
+ error: "This endpoint is not enabled for this team.",
+ });
+ expect(upstream.requests).toEqual([]);
+ });
+
+ it("returns the feed's 400 unchanged", async () => {
+ upstream.status = 400;
+ upstream.body = '{"error": "unknown parameter: colour"}';
+
+ const response = await latest("?colour=red");
+
+ expect(response.status).toBe(400);
+ expect(response.text).toBe(upstream.body);
+ });
+
+ it.each([401, 500])(
+ "answers 502 without the feed's body when the feed answers %i",
+ async status => {
+ upstream.status = status;
+ upstream.body = '{"error":"upstream detail"}';
+
+ const response = await latest();
+
+ expect(response.status).toBe(502);
+ expect(response.body).toEqual({
+ success: false,
+ error: "The request could not be completed.",
+ });
+ },
+ );
+
+ it("answers 504 when the feed does not answer in time", async () => {
+ upstream.silent = true;
+ const timeout = AbortSignal.timeout.bind(AbortSignal);
+ const shortened = vi
+ .spyOn(AbortSignal, "timeout")
+ .mockImplementation(() => timeout(50));
+
+ try {
+ const response = await latest();
+
+ expect(response.status).toBe(504);
+ expect(response.body).toEqual({
+ success: false,
+ error: "The request timed out.",
+ });
+ } finally {
+ shortened.mockRestore();
+ }
+ });
+
+ it.each([
+ [{ freshFeed: true }, false],
+ [{ freshFeed: true, searchZDR: "forced-zdr" }, true],
+ ])(
+ "logs a fresh_feed request for flags %j with zeroDataRetention %s",
+ async (flags, zeroDataRetention) => {
+ mocks.flags = flags;
+
+ await latest("?after=3");
+
+ expect(mocks.logRequest).toHaveBeenCalledExactlyOnceWith(
+ expect.objectContaining({
+ kind: "fresh_feed",
+ team_id: "team-1",
+ zeroDataRetention,
+ }),
+ );
+ },
+ );
+
+ it("compresses a large page for a caller that accepts gzip", async () => {
+ upstream.body = JSON.stringify({
+ feed: "local",
+ events: [{ text: "<p>fresh</p>".repeat(500) }],
+ });
+
+ const response = await latest().set("Accept-Encoding", "gzip");
+
+ expect(response.headers["content-encoding"]).toBe("gzip");
+ expect(response.text).toBe(upstream.body);
+ });
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index d660286da..5862f7bcb 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -144,6 +144,7 @@ const configSchema = z.object({
GOV_INDEX_CALLS_PER_DAY_OVERRIDES: teamDailyLimits,
LABS_SEARCH_URL: z.string().url().optional(),
LABS_SEARCH_SECRET: z.string().optional(),
+ FRESH_FEED_URL: emptyStringAsUndefined(z.string().url()),
// Express
EXPRESS_TRUST_PROXY: z.coerce.number().optional(),
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index a196161f6..dd8e71909 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1406,6 +1406,7 @@ export type TeamFlags = {
researchBeta?: boolean;
enrichBeta?: boolean;
labsSearch?: boolean;
+ freshFeed?: boolean;
exchangeRetrieve?: boolean;
professionalProfileCompanyDataBeta?: boolean;
// The org's DPA (or partner amendment) restricts how its data may be
diff --git a/apps/api/src/lib/__tests__/fresh-feed-config.test.ts b/apps/api/src/lib/__tests__/fresh-feed-config.test.ts
new file mode 100644
index 000000000..2b977ae6c
--- /dev/null
+++ b/apps/api/src/lib/__tests__/fresh-feed-config.test.ts
@@ -0,0 +1,15 @@
+import { afterEach, expect, it, vi } from "vitest";
+
+afterEach(() => {
+ vi.unstubAllEnvs();
+ vi.resetModules();
+});
+
+it("loads the config with an empty FRESH_FEED_URL and leaves it unset", async () => {
+ vi.stubEnv("FRESH_FEED_URL", "");
+ vi.resetModules();
+
+ const { config } = await import("../../config.js");
+
+ expect(config.FRESH_FEED_URL).toBeUndefined();
+});
diff --git a/apps/api/src/routes/fresh-feed.ts b/apps/api/src/routes/fresh-feed.ts
new file mode 100644
index 000000000..c3f0fbc0c
--- /dev/null
+++ b/apps/api/src/routes/fresh-feed.ts
@@ -0,0 +1,119 @@
+import compression from "compression";
+import express, { NextFunction, Request, Response } from "express";
+import { v7 as uuidv7 } from "uuid";
+import type { RequestWithAuth } from "../controllers/v1/types";
+import { externalRequestId } from "../lib/external-request-id";
+import { logger as rootLogger } from "../lib/logger";
+import { requestOrigin } from "../lib/request-origin";
+import { getEffectiveSearchForcedKind } from "../lib/safe-mode";
+import { logRequest } from "../services/logging/log_job";
+import { resolveIntegration } from "../utils/integration";
+import { wrap } from "./shared";
+
+const TIMEOUT_MS = 30_000;
+const UPSTREAM_PATH = "/fresh-vertical-feed/local/latest";
+
+function freshFeedError(res: Response, status: number, error: string) {
+ return res.status(status).json({ success: false, error });
+}
+
+function requireFreshFeedFlag(req: Request, res: Response, next: NextFunction) {
+ if (!(req as RequestWithAuth<any, any, any>).acuc?.flags?.freshFeed) {
+ return freshFeedError(
+ res,
+ 403,
+ "This endpoint is not enabled for this team.",
+ );
+ }
+ next();
+}
+
+function eventCount(body: Buffer): number {
+ try {
+ const events = JSON.parse(body.toString()).events;
+ return Array.isArray(events) ? events.length : 0;
+ } catch {
+ return 0;
+ }
+}
+
+function freshFeedController(upstreamBase: string) {
+ return async (req: Request, res: Response) => {
+ const authedReq = req as RequestWithAuth<any, any, any>;
+ const zeroDataRetention =
+ getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !== null;
+
+ const started = Date.now();
+ const jobId = uuidv7();
+ const logger = rootLogger.child({
+ module: "api/fresh-feed",
+ jobId,
+ teamId: authedReq.auth.team_id,
+ zeroDataRetention,
+ });
+
+ const queryStart = req.originalUrl.indexOf("?");
+ const target =
+ UPSTREAM_PATH +
+ (queryStart === -1 ? "" : req.originalUrl.slice(queryStart));
+
+ await logRequest({
+ id: jobId,
+ kind: "fresh_feed",
+ api_version: "v2",
+ external_request_id: externalRequestId(authedReq),
+ team_id: authedReq.auth.team_id,
+ origin: requestOrigin({}, req),
+ integration: resolveIntegration(undefined, req) ?? null,
+ target_hint: target,
+ zeroDataRetention,
+ api_key_id: authedReq.acuc?.api_key_id ?? null,
+ });
+
+ try {
+ const upstream = await fetch(upstreamBase + target, {
+ signal: AbortSignal.timeout(TIMEOUT_MS),
+ });
+
+ if (upstream.status !== 200 && upstream.status !== 400) {
+ await upstream.body?.cancel();
+ logger.error("Fresh feed upstream error", {
+ status: upstream.status,
+ seconds: (Date.now() - started) / 1000,
+ });
+ return freshFeedError(res, 502, "The request could not be completed.");
+ }
+
+ const body = Buffer.from(await upstream.arrayBuffer());
+ const events = upstream.status === 200 ? eventCount(body) : 0;
+
+ logger.info("Fresh feed request", {
+ status: upstream.status,
+ events,
+ seconds: (Date.now() - started) / 1000,
+ });
+
+ const contentType = upstream.headers.get("content-type");
+ if (contentType) res.setHeader("content-type", contentType);
+ return res.status(upstream.status).send(body);
+ } catch (error: unknown) {
+ if (error instanceof DOMException && error.name === "TimeoutError") {
+ logger.error("Fresh feed timed out");
+ return freshFeedError(res, 504, "The request timed out.");
+ }
+ logger.error("Fresh feed proxy error", { error });
+ return freshFeedError(res, 502, "The request could not be completed.");
+ }
+ };
+}
+
+export function createFreshFeedRouter(upstreamUrl: string) {
+ const router = express.Router();
+ router.get(
+ "/local/latest",
+ requireFreshFeedFlag,
+ compression(),
+ wrap(freshFeedController(upstreamUrl.replace(/\/+$/, ""))),
+ );
+ return router;
+}
diff --git a/apps/api/src/routes/v2.ts b/apps/api/src/routes/v2.ts
index 5c1f827b9..85806ddaf 100644
--- a/apps/api/src/routes/v2.ts
+++ b/apps/api/src/routes/v2.ts
@@ -3,6 +3,7 @@ import multer from "multer";
import { config } from "../config";
import { RateLimiterMode } from "../types";
import { registerMcpActionLogReadRoute } from "./mcp-action-logs";
+import { createFreshFeedRouter } from "./fresh-feed";
import { SEARCH_CREDITS_FEATURE_ID } from "../services/autumn/autumn.service";
import expressWs from "express-ws";
import { searchController } from "../controllers/v2/search";
@@ -777,3 +778,11 @@ if (config.SEARCH_PLATFORM_URL) {
createGovRouter(),
);
}
+
+if (config.FRESH_FEED_URL) {
+ v2Router.use(
+ "/fresh-vertical-feed",
+ authMiddleware(RateLimiterMode.FreshFeed),
+ createFreshFeedRouter(config.FRESH_FEED_URL),
+ );
+}
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 5f465c650..023e9a450 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -432,7 +432,8 @@ type LoggedRequest = {
| "research_paper_read"
| "research_related_papers"
| "code_search"
- | "gov_search";
+ | "gov_search"
+ | "fresh_feed";
api_version: string;
team_id: string;
origin?: string;
diff --git a/apps/api/src/services/rate-limiter.ts b/apps/api/src/services/rate-limiter.ts
index 507ed5e9d..212f84d31 100644
--- a/apps/api/src/services/rate-limiter.ts
+++ b/apps/api/src/services/rate-limiter.ts
@@ -42,6 +42,7 @@ const fallbackRateLimits: Record<RateLimiterMode, number> = {
exchange: 100,
// Catalogue reads only; no paid tool runs behind them.
exchangeDiscover: 10000,
+ freshFeed: 100,
};
/**
diff --git a/apps/api/src/types.ts b/apps/api/src/types.ts
index a5a0d38fb..f45172bd8 100644
--- a/apps/api/src/types.ts
+++ b/apps/api/src/types.ts
@@ -163,6 +163,7 @@ export enum RateLimiterMode {
Labs = "labs",
Exchange = "exchange",
ExchangeDiscover = "exchangeDiscover",
+ FreshFeed = "freshFeed",
}
export type AuthResponse =