Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
perf(api): relay the fresh feed's encoded response (#5748)
Karan Lokchandani71f430a2dabf
diff --git a/apps/api/src/__tests__/routes/fresh-feed.routes.test.ts b/apps/api/src/__tests__/routes/fresh-feed.routes.test.tsindex 66b640889..b72a74f92 100644--- a/apps/api/src/__tests__/routes/fresh-feed.routes.test.ts+++ b/apps/api/src/__tests__/routes/fresh-feed.routes.test.ts@@ -1,6 +1,8 @@import express from "express";+import { once } from "node:events";import http from "node:http";import type { AddressInfo } from "node:net";+import { gzipSync } from "node:zlib";import request from "supertest";const mocks = vi.hoisted(() => ({@@ -25,25 +27,46 @@ vi.mock("../../services/logging/log_job", () => ({vi.mock("../../routes/shared", () => ({ wrap: (fn: any) => fn }));+import { httpRequestDurationSeconds } from "../../lib/http-metrics";+import { logger } from "../../lib/logger";import { createFreshFeedRouter } from "../../routes/fresh-feed";+const ROUTE = "/v2/fresh-vertical-feed/local/latest";+const SHORT_TIMEOUT_ROUTE = "/short-timeout/local/latest";+const PAGE ='{"feed":"local","events":[{"n":1.0,"t":"\\u00e9"}],"next_after":7,"more":false}';const upstream = {- requests: [] as { url?: string; authorization?: string }[],+ requests: [] as {+ method?: string;+ url?: string;+ authorization?: string;+ acceptEncoding?: string;+ }[],status: 200,- body: PAGE,+ headers: {} as http.OutgoingHttpHeaders,+ body: PAGE as string | Buffer,silent: false,+ stallAfter: undefined as number | undefined,};const upstreamServer = http.createServer((req, res) => {upstream.requests.push({+ method: req.method,url: req.url,authorization: req.headers.authorization,+ acceptEncoding: req.headers["accept-encoding"],});if (upstream.silent) return;- res.writeHead(upstream.status, { "content-type": "application/json" });+ res.writeHead(req.url === "/moved" ? 200 : upstream.status, {+ "content-type": "application/json",+ ...upstream.headers,+ });+ if (upstream.stallAfter !== undefined) {+ res.write(upstream.body.slice(0, upstream.stallAfter));+ return;+ }res.end(upstream.body);});@@ -54,25 +77,61 @@ app.use((req: any, _res, next) => {next();});-const latest = (query = "") =>+const latest = (query = "", route = ROUTE) =>request(app)- .get("/v2/fresh-vertical-feed/local/latest" + query)+ .get(route + query).set("Authorization", "Bearer fc-caller-key");+const apiServer = http.createServer(app);++function undecoded(call: {+ method?: string;+ route?: string;+ query?: string;+ headers?: http.OutgoingHttpHeaders;+}) {+ return new Promise<{ response: http.IncomingMessage; body: Buffer }>(+ (resolve, reject) => {+ http+ .request(+ {+ host: "127.0.0.1",+ port: (apiServer.address() as AddressInfo).port,+ path: (call.route ?? ROUTE) + (call.query ?? ""),+ method: call.method,+ headers: call.headers,+ },+ response => {+ const chunks: Buffer[] = [];+ response.on("data", chunk => chunks.push(chunk));+ response.on("error", () => {});+ response.on("close", () =>+ resolve({ response, body: Buffer.concat(chunks) }),+ );+ },+ )+ .on("error", reject)+ .end();+ },+ );+}+beforeAll(async () => {await new Promise<void>(resolve =>upstreamServer.listen(0, "127.0.0.1", resolve),);const { port } = upstreamServer.address() as AddressInfo;- app.use(- "/v2/fresh-vertical-feed",- createFreshFeedRouter(`http://127.0.0.1:${port}/`),- );+ const feedUrl = `http://127.0.0.1:${port}/`;+ app.use("/v2/fresh-vertical-feed", createFreshFeedRouter(feedUrl));+ app.use("/short-timeout", createFreshFeedRouter(feedUrl, 50));+ await new Promise<void>(resolve => apiServer.listen(0, "127.0.0.1", resolve));});afterAll(async () => {- upstreamServer.closeAllConnections();- await new Promise(resolve => upstreamServer.close(resolve));+ for (const server of [apiServer, upstreamServer]) {+ server.closeAllConnections();+ await new Promise(resolve => server.close(resolve));+ }});beforeEach(() => {@@ -81,24 +140,31 @@ beforeEach(() => {mocks.flags = { freshFeed: true };upstream.requests.length = 0;upstream.status = 200;+ upstream.headers = {};upstream.body = PAGE;upstream.silent = false;+ upstream.stallAfter = undefined;});describe("GET /v2/fresh-vertical-feed/local/latest", () => {it("calls the feed with the caller's raw query and without the caller's key, and returns its body byte for byte", async () => {const query = "?limit=5&after=12&host=b+c&host=caf%C3%A9.example&x=";- const response = await latest(query);+ const { response, body } = await undecoded({+ query,+ headers: { authorization: "Bearer fc-caller-key" },+ });expect(upstream.requests).toEqual([{+ method: "GET",url: "/fresh-vertical-feed/local/latest" + query,authorization: undefined,+ acceptEncoding: undefined,},]);- expect(response.status).toBe(200);- expect(response.text).toBe(PAGE);+ expect(response.statusCode).toBe(200);+ expect(body.toString()).toBe(PAGE);});it("refuses a team without the flag before the feed is called", async () => {@@ -114,52 +180,106 @@ describe("GET /v2/fresh-vertical-feed/local/latest", () => {expect(upstream.requests).toEqual([]);});- it("returns the feed's 400 unchanged", async () => {- upstream.status = 400;+ it("refuses a team whose flag is not the boolean true", async () => {+ mocks.flags = { freshFeed: "false" };++ const response = await latest("?after=0");++ expect(response.status).toBe(403);+ expect(upstream.requests).toEqual([]);+ });++ it.each([400, 406])("returns the feed's %i unchanged", async status => {+ upstream.status = status;upstream.body = '{"error": "unknown parameter: colour"}';const response = await latest("?colour=red");- expect(response.status).toBe(400);+ expect(response.status).toBe(status);expect(response.text).toBe(upstream.body);});- it.each([401, 500])(- "answers 502 without the feed's body when the feed answers %i",+ it("answers 502 without the feed's body when the feed answers 401", async () => {+ upstream.status = 401;+ upstream.body = '{"error":"upstream detail"}';++ const response = await latest();++ expect(response.status).toBe(502);+ expect(response.headers["retry-after"]).toBeUndefined();+ expect(response.body).toEqual({+ success: false,+ error: "The request could not be completed.",+ });+ });++ it.each([500, 503])(+ "answers 503 with Retry-After and without the feed's body when the feed answers %i",async status => {upstream.status = status;upstream.body = '{"error":"upstream detail"}';const response = await latest();- expect(response.status).toBe(502);+ expect(response.status).toBe(503);+ expect(response.headers["retry-after"]).toBe("5");expect(response.body).toEqual({success: false,- error: "The request could not be completed.",+ error: "The feed is temporarily unavailable. Retry the same request.",});},);- it("answers 504 when the feed does not answer in time", async () => {- upstream.silent = true;- const timeout = AbortSignal.timeout.bind(AbortSignal);- const shortened = vi- .spyOn(AbortSignal, "timeout")- .mockImplementation(() => timeout(50));+ it("answers 503 without waiting for the rest of the feed's error body", async () => {+ upstream.status = 500;+ upstream.stallAfter = 5;- try {- const response = await latest();+ const response = await latest();- expect(response.status).toBe(504);- expect(response.body).toEqual({- success: false,- error: "The request timed out.",- });- } finally {- shortened.mockRestore();- }+ expect(response.status).toBe(503);});+ it("answers 502 to a redirect from the feed without following it", async () => {+ upstream.status = 302;+ upstream.headers = { location: "/moved" };++ const response = await latest();++ expect(response.status).toBe(502);+ expect(upstream.requests).toHaveLength(1);+ });++ it("answers 504 when the feed sends no headers within the timeout", async () => {+ upstream.silent = true;++ const response = await latest("", SHORT_TIMEOUT_ROUTE);++ expect(response.status).toBe(504);+ expect(response.body).toEqual({+ success: false,+ error: "The request timed out.",+ });+ }, 10_000);++ it("stops the feed call without logging an error when the caller hangs up before the feed answers", async () => {+ upstream.silent = true;+ const reachedFeed = once(upstreamServer, "request");+ const call = http+ .get({+ host: "127.0.0.1",+ port: (apiServer.address() as AddressInfo).port,+ path: ROUTE,+ })+ .on("error", () => {});+ const [feedRequest] = (await reachedFeed) as [http.IncomingMessage];+ const feedCallStopped = once(feedRequest.socket, "close");++ call.destroy();+ await feedCallStopped;++ expect(logger.error).not.toHaveBeenCalled();+ }, 10_000);+it.each([[{ freshFeed: true }, false],[{ freshFeed: true, searchZDR: "forced-zdr" }, true],@@ -191,4 +311,82 @@ describe("GET /v2/fresh-vertical-feed/local/latest", () => {expect(response.headers["content-encoding"]).toBe("gzip");expect(response.text).toBe(upstream.body);});++ it("passes the caller's Accept-Encoding to the feed and relays the feed's encoded bytes untouched", async () => {+ const encoded = gzipSync("<p>fresh</p>".repeat(500), { level: 1 });+ upstream.body = encoded;+ upstream.headers = {+ "content-encoding": "gzip",+ "content-length": encoded.length,+ vary: "Accept-Encoding, Origin",+ "set-cookie": "feed=1",+ "x-feed-only": "1",+ };++ const { response, body } = await undecoded({+ headers: { "accept-encoding": "gzip;q=0.9, zstd" },+ });++ expect(body.equals(encoded)).toBe(true);+ expect(upstream.requests[0].acceptEncoding).toBe("gzip;q=0.9, zstd");+ expect(response.headers).toMatchObject({+ "content-encoding": "gzip",+ "content-length": String(encoded.length),+ vary: "Accept-Encoding, Origin",+ });+ expect(response.headers).not.toHaveProperty("set-cookie");+ expect(response.headers).not.toHaveProperty("x-feed-only");+ });++ it("answers 405 to a HEAD before the request is logged or the feed is called", async () => {+ const { response } = await undecoded({ method: "HEAD" });++ expect(response.statusCode).toBe(405);+ expect(response.headers.allow).toBe("GET");+ expect(upstream.requests).toEqual([]);+ expect(mocks.logRequest).not.toHaveBeenCalled();+ });++ it("breaks the caller's transfer when the feed goes silent partway through the body, and still records its duration", async () => {+ upstream.stallAfter = 20;+ httpRequestDurationSeconds.reset();++ const { response, body } = await undecoded({ route: SHORT_TIMEOUT_ROUTE });++ expect(body.toString()).toBe(PAGE.slice(0, 20));+ expect(response.complete).toBe(false);+ await vi.waitFor(async () => {+ const { values } = await httpRequestDurationSeconds.get();+ expect(+ values.filter(+ v => v.metricName === "http_request_duration_seconds_count",+ ),+ ).toHaveLength(1);+ });+ }, 10_000);++ it("records a relayed response in the request duration histogram", async () => {+ httpRequestDurationSeconds.reset();++ await latest();++ await vi.waitFor(async () => {+ const { values } = await httpRequestDurationSeconds.get();+ expect(+ values+ .filter(v => v.metricName === "http_request_duration_seconds_count")+ .map(v => [v.labels, v.value]),+ ).toEqual([+ [+ {+ version: "v2",+ method: "GET",+ route: "/local/latest",+ status: "200",+ },+ 1,+ ],+ ]);+ });+ });});diff --git a/apps/api/src/routes/fresh-feed.ts b/apps/api/src/routes/fresh-feed.tsindex c3f0fbc0c..037811d05 100644--- a/apps/api/src/routes/fresh-feed.ts+++ b/apps/api/src/routes/fresh-feed.ts@@ -1,8 +1,14 @@import compression from "compression";import express, { NextFunction, Request, Response } from "express";+import { pipeline } from "node:stream/promises";+import { request, type Dispatcher } from "undici";import { v7 as uuidv7 } from "uuid";import type { RequestWithAuth } from "../controllers/v1/types";import { externalRequestId } from "../lib/external-request-id";+import {+ getRoutePattern,+ httpRequestDurationSeconds,+} from "../lib/http-metrics";import { logger as rootLogger } from "../lib/logger";import { requestOrigin } from "../lib/request-origin";import { getEffectiveSearchForcedKind } from "../lib/safe-mode";@@ -12,13 +18,19 @@ import { wrap } from "./shared";const TIMEOUT_MS = 30_000;const UPSTREAM_PATH = "/fresh-vertical-feed/local/latest";+const FORWARDED_RESPONSE_HEADERS = [+ "content-type",+ "content-encoding",+ "content-length",+ "vary",+];function freshFeedError(res: Response, status: number, error: string) {return res.status(status).json({ success: false, error });}function requireFreshFeedFlag(req: Request, res: Response, next: NextFunction) {- if (!(req as RequestWithAuth<any, any, any>).acuc?.flags?.freshFeed) {+ if ((req as RequestWithAuth<any, any, any>).acuc?.flags?.freshFeed !== true) {return freshFeedError(res,403,@@ -28,22 +40,17 @@ function requireFreshFeedFlag(req: Request, res: Response, next: NextFunction) {next();}-function eventCount(body: Buffer): number {- try {- const events = JSON.parse(body.toString()).events;- return Array.isArray(events) ? events.length : 0;- } catch {- return 0;- }+function rejectHead(_req: Request, res: Response) {+ res.status(405).set("Allow", "GET").end();}-function freshFeedController(upstreamBase: string) {+function freshFeedController(upstreamBase: string, timeoutMs: number) {return async (req: Request, res: Response) => {const authedReq = req as RequestWithAuth<any, any, any>;const zeroDataRetention =getEffectiveSearchForcedKind(authedReq.acuc?.flags, undefined) !== null;- const started = Date.now();+ const started: number = (req as any).requestTiming?.startTime ?? Date.now();const jobId = uuidv7();const logger = rootLogger.child({module: "api/fresh-feed",@@ -57,6 +64,9 @@ function freshFeedController(upstreamBase: string) {UPSTREAM_PATH +(queryStart === -1 ? "" : req.originalUrl.slice(queryStart));+ const callerGone = new AbortController();+ res.on("close", () => callerGone.abort());+await logRequest({id: jobId,kind: "fresh_feed",@@ -70,50 +80,82 @@ function freshFeedController(upstreamBase: string) {api_key_id: authedReq.acuc?.api_key_id ?? null,});+ let upstream: Dispatcher.ResponseData;try {- const upstream = await fetch(upstreamBase + target, {- signal: AbortSignal.timeout(TIMEOUT_MS),- });-- if (upstream.status !== 200 && upstream.status !== 400) {- await upstream.body?.cancel();- logger.error("Fresh feed upstream error", {- status: upstream.status,- seconds: (Date.now() - started) / 1000,- });- return freshFeedError(res, 502, "The request could not be completed.");- }-- const body = Buffer.from(await upstream.arrayBuffer());- const events = upstream.status === 200 ? eventCount(body) : 0;-- logger.info("Fresh feed request", {- status: upstream.status,- events,- seconds: (Date.now() - started) / 1000,+ // fetch always decodes the body; request hands back the feed's bytes as sent.+ upstream = await request(upstreamBase + target, {+ headers: { "accept-encoding": req.headers["accept-encoding"] },+ headersTimeout: timeoutMs,+ bodyTimeout: timeoutMs,+ signal: callerGone.signal,});-- const contentType = upstream.headers.get("content-type");- if (contentType) res.setHeader("content-type", contentType);- return res.status(upstream.status).send(body);} catch (error: unknown) {- if (error instanceof DOMException && error.name === "TimeoutError") {+ if (callerGone.signal.aborted) return;+ if ((error as { code?: unknown })?.code === "UND_ERR_HEADERS_TIMEOUT") {logger.error("Fresh feed timed out");return freshFeedError(res, 504, "The request timed out.");}logger.error("Fresh feed proxy error", { error });return freshFeedError(res, 502, "The request could not be completed.");}++ if (![200, 400, 406].includes(upstream.statusCode)) {+ upstream.body.dump().catch(() => {});+ logger.error("Fresh feed upstream error", {+ status: upstream.statusCode,+ seconds: (Date.now() - started) / 1000,+ });+ if (upstream.statusCode === 500 || upstream.statusCode === 503) {+ res.setHeader("Retry-After", "5");+ return freshFeedError(+ res,+ 503,+ "The feed is temporarily unavailable. Retry the same request.",+ );+ }+ return freshFeedError(res, 502, "The request could not be completed.");+ }++ res.status(upstream.statusCode);+ for (const h of FORWARDED_RESPONSE_HEADERS) {+ const value = upstream.headers[h];+ if (value) res.setHeader(h, value);+ }++ let bytes = 0;+ upstream.body.on("data", (chunk: Buffer) => {+ bytes += chunk.length;+ });+ const outcome = () => ({+ status: upstream.statusCode,+ contentEncoding: upstream.headers["content-encoding"],+ bytes,+ seconds: (Date.now() - started) / 1000,+ });++ try {+ await pipeline(upstream.body, res);+ logger.info("Fresh feed request", outcome());+ } catch (error: unknown) {+ logger.warn("Fresh feed stream failed", { error, ...outcome() });+ }+ httpRequestDurationSeconds+ .labels("v2", req.method, getRoutePattern(req), String(res.statusCode))+ .observe((Date.now() - started) / 1000);};}-export function createFreshFeedRouter(upstreamUrl: string) {+export function createFreshFeedRouter(+ upstreamUrl: string,+ timeoutMs = TIMEOUT_MS,+) {const router = express.Router();+ router.head("/local/latest", rejectHead);router.get("/local/latest",requireFreshFeedFlag,compression(),- wrap(freshFeedController(upstreamUrl.replace(/\/+$/, ""))),+ wrap(freshFeedController(upstreamUrl.replace(/\/+$/, ""), timeoutMs)),);return router;}