Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): add bounded highlights shadow traffic (#3996)
Gergő Móricz0643a6831e81
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 8a672944d..112608208 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -163,6 +163,8 @@ const configSchema = z.object({// bearer auth for legacy/external services; the in-cluster service omits it.HIGHLIGHT_MODEL_URL: z.string().optional(),HIGHLIGHT_MODEL_TOKEN: z.string().optional(),+ HIGHLIGHT_SHADOW_RATE: z.coerce.number().min(0).max(1).default(0),+ HIGHLIGHT_SHADOW_MAX_INFLIGHT: z.coerce.number().int().positive().default(8),// Fire EngineFIRE_ENGINE_BETA_URL: z.string().optional(),diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex caabca548..6927e37d1 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -14,6 +14,7 @@ import {calculateScrapeCredits,} from "./scrape";import { applySearchHighlights, highlightsEnvReady } from "./highlights";+import { runSearchHighlightsShadow } from "./highlights-shadow";import { trackSearchResults, trackSearchRequest } from "../lib/tracking";import type { BillingMetadata } from "../services/billing/types";import type { ThreatProtectionPolicy } from "../lib/threat-protection/types";@@ -247,12 +248,21 @@ export async function executeSearch(// Runs after scraping (mergeScrapedContent rebuilds the result objects, so// highlight mutations must come last to survive). Uses the user's original// query, not the domain-filtered upstream query.- if (+ const shouldApplyHighlights =options.highlights &&flags?.highlightsBeta === true &&- highlightsEnvReady()- ) {+ highlightsEnvReady();+ if (shouldApplyHighlights) {await applySearchHighlights(searchResponse, query, logger);+ } else {+ runSearchHighlightsShadow({+ response: searchResponse,+ query,+ requestId: context.requestId,+ teamId,+ zeroDataRetention: zeroDataRetention === true || isZDR === true,+ logger,+ });}const scrapeFormats = scrapeOptions?.formatsdiff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.tsindex b70925d8a..7881ad985 100644--- a/apps/api/src/search/highlight-model.test.ts+++ b/apps/api/src/search/highlight-model.test.ts@@ -227,4 +227,20 @@ describe("generateHighlightsBatch", () => {expect.objectContaining({ pageId: "1" }),);});++ it("does not fan out fallback requests when fallback is disabled", async () => {+ const fetchMock = mockFetchOnce({ error: "not found" }, false, 404);++ const out = await generateHighlightsBatch(+ "q",+ [+ { id: "0", markdown: "first" },+ { id: "1", markdown: "second" },+ ],+ { logger, allowLegacyFallback: false, logPayload: false },+ );++ expect(fetchMock).toHaveBeenCalledTimes(1);+ expect(out).toBeNull();+ });});diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.tsindex 1feb53c09..b3481eac3 100644--- a/apps/api/src/search/highlight-model.ts+++ b/apps/api/src/search/highlight-model.ts@@ -66,7 +66,11 @@ function requestHeaders(): Record<string, string> {export async function generateHighlightsBatch(query: string,pages: HighlightPage[],- opts: { logger: Logger },+ opts: {+ logger: Logger;+ logPayload?: boolean;+ allowLegacyFallback?: boolean;+ },): Promise<Map<string, HighlightResult> | null> {if (pages.length === 0) {return new Map();@@ -89,7 +93,10 @@ export async function generateHighlightsBatch(// During rollout the configured service may still be the legacy Modal// endpoint, whose /batch_highlight contract uses {requests: [...]}. Keep// highlights available until infra switches the URL to GCP Stage 1.- if (res.status === 400 || res.status === 404) {+ if (+ opts.allowLegacyFallback !== false &&+ (res.status === 400 || res.status === 404)+ ) {opts.logger.info("query highlights using legacy per-page fallback", {canonicalLog: "search/highlights",status: res.status,@@ -121,10 +128,14 @@ export async function generateHighlightsBatch(canonicalLog: "search/highlights",requestedPages: pages.length,returnedPages: results.size,- pages: Array.from(results, ([id, result]) => ({- id,- highlights: result.highlights,- })),+ ...(opts.logPayload === false+ ? {}+ : {+ pages: Array.from(results, ([id, result]) => ({+ id,+ highlights: result.highlights,+ })),+ }),elapsedMs: Date.now() - start,});diff --git a/apps/api/src/search/highlights-shadow.test.ts b/apps/api/src/search/highlights-shadow.test.tsnew file mode 100644index 000000000..b89acd37d--- /dev/null+++ b/apps/api/src/search/highlights-shadow.test.ts@@ -0,0 +1,145 @@+vi.mock("../config", () => ({+ config: {+ HIGHLIGHT_SHADOW_RATE: 1,+ HIGHLIGHT_SHADOW_MAX_INFLIGHT: 1,+ },+}));++vi.mock("./highlights", () => ({+ highlightsEnvReady: vi.fn(() => true),+ applySearchHighlights: vi.fn(),+}));++import { config } from "../config";+import { applySearchHighlights, highlightsEnvReady } from "./highlights";+import { createSearchHighlightsShadowRunner } from "./highlights-shadow";++const logger = {+ info: vi.fn(),+ warn: vi.fn(),+} as any;++let runSearchHighlightsShadow = createSearchHighlightsShadowRunner();++afterEach(() => {+ vi.clearAllMocks();+ config.HIGHLIGHT_SHADOW_RATE = 1;+ config.HIGHLIGHT_SHADOW_MAX_INFLIGHT = 1;+ vi.mocked(highlightsEnvReady).mockReturnValue(true);+ runSearchHighlightsShadow = createSearchHighlightsShadowRunner();+});++describe("runSearchHighlightsShadow", () => {+ it("runs without applying results and emits a content-free canonical log", async () => {+ vi.mocked(applySearchHighlights).mockResolvedValue({+ attempted: 10,+ indexHits: 7,+ replaced: 6,+ succeeded: true,+ });++ expect(+ runSearchHighlightsShadow({+ response: {} as any,+ query: "private query",+ requestId: "request-1",+ teamId: "team-1",+ zeroDataRetention: false,+ logger,+ }),+ ).toBe("started");+ await new Promise(resolve => setImmediate(resolve));++ expect(applySearchHighlights).toHaveBeenCalledWith(+ {},+ "private query",+ expect.objectContaining({ silent: true }),+ {+ applyResults: false,+ suppressSummaryLog: true,+ suppressPayloadLog: true,+ allowLegacyFallback: false,+ },+ );+ expect(logger.info).toHaveBeenCalledWith(+ "Search highlights shadow completed",+ expect.objectContaining({+ canonicalLog: "search/highlights-shadow",+ outcome: "completed",+ attempted: 10,+ indexHits: 7,+ wouldReplace: 6,+ }),+ );+ const fields = logger.info.mock.calls[0][1];+ expect(fields).not.toHaveProperty("query");+ expect(fields).not.toHaveProperty("markdown");+ expect(fields).not.toHaveProperty("highlights");+ });++ it("drops excess work instead of creating a backlog", async () => {+ let finish!: (value: {+ attempted: number;+ indexHits: number;+ replaced: number;+ succeeded: boolean;+ }) => void;+ vi.mocked(applySearchHighlights).mockReturnValue(+ new Promise(resolve => {+ finish = resolve;+ }),+ );++ const input = {+ response: {} as any,+ query: "query",+ teamId: "team-1",+ zeroDataRetention: false,+ logger,+ };+ expect(+ runSearchHighlightsShadow({ ...input, requestId: "request-1" }),+ ).toBe("started");+ expect(+ runSearchHighlightsShadow({ ...input, requestId: "request-2" }),+ ).toBe("dropped");+ expect(logger.info).toHaveBeenCalledWith(+ "Search highlights shadow dropped",+ expect.objectContaining({+ canonicalLog: "search/highlights-shadow",+ outcome: "dropped",+ reason: "max_inflight",+ }),+ );++ finish({ attempted: 1, indexHits: 1, replaced: 1, succeeded: true });+ await new Promise(resolve => setImmediate(resolve));+ });++ it("does not shadow ZDR, disabled, or unavailable requests", () => {+ const input = {+ response: {} as any,+ query: "query",+ requestId: "request-1",+ teamId: "team-1",+ logger,+ };++ expect(+ runSearchHighlightsShadow({ ...input, zeroDataRetention: true }),+ ).toBe("skipped");++ config.HIGHLIGHT_SHADOW_RATE = 0;+ expect(+ runSearchHighlightsShadow({ ...input, zeroDataRetention: false }),+ ).toBe("skipped");++ config.HIGHLIGHT_SHADOW_RATE = 1;+ vi.mocked(highlightsEnvReady).mockReturnValue(false);+ expect(+ runSearchHighlightsShadow({ ...input, zeroDataRetention: false }),+ ).toBe("skipped");++ expect(applySearchHighlights).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/search/highlights-shadow.ts b/apps/api/src/search/highlights-shadow.tsnew file mode 100644index 000000000..2479ee91d--- /dev/null+++ b/apps/api/src/search/highlights-shadow.ts@@ -0,0 +1,98 @@+import { createLogger, type Logger } from "winston";+import type { SearchV2Response } from "../lib/entities";+import { config } from "../config";+import { applySearchHighlights, highlightsEnvReady } from "./highlights";++const CANONICAL_LOG = "search/highlights-shadow";+const shadowLogger = createLogger({ silent: true });++function sampled(requestId: string, rate: number): boolean {+ if (rate <= 0) return false;+ if (rate >= 1) return true;++ let hash = 2166136261;+ for (let i = 0; i < requestId.length; i++) {+ hash ^= requestId.charCodeAt(i);+ hash = Math.imul(hash, 16777619);+ }+ return (hash >>> 0) / 0x1_0000_0000 < rate;+}++export function createSearchHighlightsShadowRunner(): (options: {+ response: SearchV2Response;+ query: string;+ requestId: string;+ teamId: string;+ zeroDataRetention: boolean;+ logger: Logger;+}) => "skipped" | "dropped" | "started" {+ let inFlight = 0;++ return options => {+ if (+ options.zeroDataRetention ||+ !highlightsEnvReady() ||+ !sampled(options.requestId, config.HIGHLIGHT_SHADOW_RATE)+ ) {+ return "skipped";+ }++ if (inFlight >= config.HIGHLIGHT_SHADOW_MAX_INFLIGHT) {+ options.logger.info("Search highlights shadow dropped", {+ canonicalLog: CANONICAL_LOG,+ outcome: "dropped",+ reason: "max_inflight",+ requestId: options.requestId,+ teamId: options.teamId,+ inFlight,+ maxInFlight: config.HIGHLIGHT_SHADOW_MAX_INFLIGHT,+ sampleRate: config.HIGHLIGHT_SHADOW_RATE,+ });+ return "dropped";+ }++ inFlight++;+ const startedAt = Date.now();+ void applySearchHighlights(options.response, options.query, shadowLogger, {+ applyResults: false,+ suppressSummaryLog: true,+ suppressPayloadLog: true,+ allowLegacyFallback: false,+ })+ .then(result => {+ options.logger.info("Search highlights shadow completed", {+ canonicalLog: CANONICAL_LOG,+ outcome: result.succeeded ? "completed" : "failed",+ requestId: options.requestId,+ teamId: options.teamId,+ attempted: result.attempted,+ indexHits: result.indexHits,+ wouldReplace: result.replaced,+ timeTakenMs: Date.now() - startedAt,+ inFlight,+ maxInFlight: config.HIGHLIGHT_SHADOW_MAX_INFLIGHT,+ sampleRate: config.HIGHLIGHT_SHADOW_RATE,+ });+ })+ .catch(error => {+ options.logger.warn("Search highlights shadow failed", {+ canonicalLog: CANONICAL_LOG,+ outcome: "failed",+ requestId: options.requestId,+ teamId: options.teamId,+ errorType: error instanceof Error ? error.name : "unknown",+ timeTakenMs: Date.now() - startedAt,+ inFlight,+ maxInFlight: config.HIGHLIGHT_SHADOW_MAX_INFLIGHT,+ sampleRate: config.HIGHLIGHT_SHADOW_RATE,+ });+ })+ .finally(() => {+ inFlight--;+ });++ return "started";+ };+}++export const runSearchHighlightsShadow = createSearchHighlightsShadowRunner();diff --git a/apps/api/src/search/highlights.test.ts b/apps/api/src/search/highlights.test.tsindex ccaa8791f..821a5017d 100644--- a/apps/api/src/search/highlights.test.ts+++ b/apps/api/src/search/highlights.test.ts@@ -39,6 +39,7 @@ vi.mock("./highlight-model", () => ({import { generateHighlightsBatch } from "./highlight-model";import { config } from "../config";+import { indexGetRecent5 } from "../db/rpc";import { applySearchHighlights, highlightsEnvReady } from "./highlights";const logger = {@@ -98,7 +99,12 @@ describe("applySearchHighlights", () => {);expect(response.web[0].description).toBe("first highlight");expect(response.news[0].snippet).toBe("second highlight");- expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 2 });+ expect(result).toEqual({+ attempted: 2,+ indexHits: 2,+ replaced: 2,+ succeeded: true,+ });});it("preserves individual fallbacks for missing and empty batch pages", async () => {@@ -118,7 +124,12 @@ describe("applySearchHighlights", () => {"first fallback","second fallback",]);- expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 0 });+ expect(result).toEqual({+ attempted: 2,+ indexHits: 2,+ replaced: 0,+ succeeded: true,+ });});it("preserves every fallback when the batch request fails", async () => {@@ -136,6 +147,71 @@ describe("applySearchHighlights", () => {"first fallback","second fallback",]);- expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 0 });+ expect(result).toEqual({+ attempted: 2,+ indexHits: 2,+ replaced: 0,+ succeeded: false,+ });+ });++ it("runs without mutating the response or logging payloads in shadow mode", async () => {+ vi.mocked(generateHighlightsBatch).mockResolvedValue(+ new Map([["0", { highlights: [], markdown: "shadow highlight" }]]),+ );+ const response = {+ web: [{ url: "https://first.test", description: "fallback" }],+ } as any;++ const result = await applySearchHighlights(response, "query", logger, {+ applyResults: false,+ suppressSummaryLog: true,+ suppressPayloadLog: true,+ allowLegacyFallback: false,+ });++ expect(response.web[0].description).toBe("fallback");+ expect(result).toEqual({+ attempted: 1,+ indexHits: 1,+ replaced: 1,+ succeeded: true,+ });+ expect(generateHighlightsBatch).toHaveBeenCalledWith(+ "query",+ expect.any(Array),+ {+ logger,+ logPayload: false,+ allowLegacyFallback: false,+ },+ );+ expect(logger.info).not.toHaveBeenCalledWith(+ "Search highlights applied",+ expect.anything(),+ );+ });++ it("omits result URLs from shadow lookup failures", async () => {+ vi.mocked(indexGetRecent5).mockRejectedValueOnce(+ new Error("lookup failed"),+ );+ const response = {+ web: [{ url: "https://private.test", description: "fallback" }],+ } as any;++ await applySearchHighlights(response, "query", logger, {+ applyResults: false,+ suppressSummaryLog: true,+ suppressPayloadLog: true,+ allowLegacyFallback: false,+ });++ expect(logger.warn).toHaveBeenCalledWith(+ "highlights: index lookup failed",+ {+ error: "lookup failed",+ },+ );});});diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.tsindex 62a8a0c3b..f04e9d6c6 100644--- a/apps/api/src/search/highlights.ts+++ b/apps/api/src/search/highlights.ts@@ -44,6 +44,7 @@ const ERROR_COUNT_TO_REGISTER = 3;async function getIndexedMarkdownForURL(url: string,logger: Logger,+ logUrl = true,): Promise<string | null> {if (!useIndex) {return null;@@ -110,7 +111,7 @@ async function getIndexedMarkdownForURL(} catch (error) {logger.warn("highlights: index lookup failed", {error: error instanceof Error ? error.message : String(error),- url,+ ...(logUrl ? { url } : {}),});return null;}@@ -129,8 +130,20 @@ export async function applySearchHighlights(response: SearchV2Response,query: string,logger: Logger,-): Promise<{ attempted: number; indexHits: number; replaced: number }> {+ options: {+ applyResults?: boolean;+ suppressSummaryLog?: boolean;+ suppressPayloadLog?: boolean;+ allowLegacyFallback?: boolean;+ } = {},+): Promise<{+ attempted: number;+ indexHits: number;+ replaced: number;+ succeeded: boolean;+}> {const start = Date.now();+ const applyResults = options.applyResults ?? true;// Collect every result we could highlight, each with a setter for its snippet// field: web results carry it in `description`, news results in `snippet`.@@ -156,13 +169,15 @@ export async function applySearchHighlights(const attempted = targets.length;if (attempted === 0) {- return { attempted, indexHits: 0, replaced: 0 };+ return { attempted, indexHits: 0, replaced: 0, succeeded: true };}// Look up indexed markdown for every URL in parallel, keeping the markdown for// each hit so we can send it to the highlight model service.const markdowns = await Promise.all(- targets.map(t => getIndexedMarkdownForURL(t.url, logger)),+ targets.map(t =>+ getIndexedMarkdownForURL(t.url, logger, !options.suppressPayloadLog),+ ),);const hits: {apply: (h: string) => void;@@ -177,6 +192,7 @@ export async function applySearchHighlights(// Send every hit in one request. IDs are local batch indexes, so a missing or// empty response only falls back the corresponding provider snippet.let replaced = 0;+ let succeeded = true;if (indexHits > 0) {const results = await generateHighlightsBatch(query,@@ -184,25 +200,41 @@ export async function applySearchHighlights(id: String(index),markdown: hit.markdown,})),- { logger },+ options.suppressPayloadLog || options.allowLegacyFallback === false+ ? {+ logger,+ logPayload: !options.suppressPayloadLog,+ allowLegacyFallback: options.allowLegacyFallback,+ }+ : { logger },);+ succeeded = results !== null;if (results) {hits.forEach((hit, index) => {const snippet = results.get(String(index))?.markdown;if (snippet?.trim()) {- hit.apply(snippet);+ if (applyResults) {+ hit.apply(snippet);+ }replaced++;}});}}- logger.info("Search highlights applied", {+ if (!options.suppressSummaryLog) {+ logger.info("Search highlights applied", {+ attempted,+ indexHits,+ replaced,+ timeTakenMs: Date.now() - start,+ });+ }++ return {attempted,indexHits,replaced,- timeTakenMs: Date.now() - start,- });-- return { attempted, indexHits, replaced };+ succeeded,+ };}