Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(search): add bounded highlights shadow traffic (#3996)

Gergő Móricz0643a6831e81
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 8a672944d..112608208 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -163,6 +163,8 @@ const configSchema = z.object({
// bearer auth for legacy/external services; the in-cluster service omits it.
HIGHLIGHT_MODEL_URL: z.string().optional(),
HIGHLIGHT_MODEL_TOKEN: z.string().optional(),
+ HIGHLIGHT_SHADOW_RATE: z.coerce.number().min(0).max(1).default(0),
+ HIGHLIGHT_SHADOW_MAX_INFLIGHT: z.coerce.number().int().positive().default(8),
// Fire Engine
FIRE_ENGINE_BETA_URL: z.string().optional(),
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index caabca548..6927e37d1 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -14,6 +14,7 @@ import {
calculateScrapeCredits,
} from "./scrape";
import { applySearchHighlights, highlightsEnvReady } from "./highlights";
+import { runSearchHighlightsShadow } from "./highlights-shadow";
import { trackSearchResults, trackSearchRequest } from "../lib/tracking";
import type { BillingMetadata } from "../services/billing/types";
import type { ThreatProtectionPolicy } from "../lib/threat-protection/types";
@@ -247,12 +248,21 @@ export async function executeSearch(
// Runs after scraping (mergeScrapedContent rebuilds the result objects, so
// highlight mutations must come last to survive). Uses the user's original
// query, not the domain-filtered upstream query.
- if (
+ const shouldApplyHighlights =
options.highlights &&
flags?.highlightsBeta === true &&
- highlightsEnvReady()
- ) {
+ highlightsEnvReady();
+ if (shouldApplyHighlights) {
await applySearchHighlights(searchResponse, query, logger);
+ } else {
+ runSearchHighlightsShadow({
+ response: searchResponse,
+ query,
+ requestId: context.requestId,
+ teamId,
+ zeroDataRetention: zeroDataRetention === true || isZDR === true,
+ logger,
+ });
}
const scrapeFormats = scrapeOptions?.formats
diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.ts
index b70925d8a..7881ad985 100644
--- a/apps/api/src/search/highlight-model.test.ts
+++ b/apps/api/src/search/highlight-model.test.ts
@@ -227,4 +227,20 @@ describe("generateHighlightsBatch", () => {
expect.objectContaining({ pageId: "1" }),
);
});
+
+ it("does not fan out fallback requests when fallback is disabled", async () => {
+ const fetchMock = mockFetchOnce({ error: "not found" }, false, 404);
+
+ const out = await generateHighlightsBatch(
+ "q",
+ [
+ { id: "0", markdown: "first" },
+ { id: "1", markdown: "second" },
+ ],
+ { logger, allowLegacyFallback: false, logPayload: false },
+ );
+
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ expect(out).toBeNull();
+ });
});
diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.ts
index 1feb53c09..b3481eac3 100644
--- a/apps/api/src/search/highlight-model.ts
+++ b/apps/api/src/search/highlight-model.ts
@@ -66,7 +66,11 @@ function requestHeaders(): Record<string, string> {
export async function generateHighlightsBatch(
query: string,
pages: HighlightPage[],
- opts: { logger: Logger },
+ opts: {
+ logger: Logger;
+ logPayload?: boolean;
+ allowLegacyFallback?: boolean;
+ },
): Promise<Map<string, HighlightResult> | null> {
if (pages.length === 0) {
return new Map();
@@ -89,7 +93,10 @@ export async function generateHighlightsBatch(
// During rollout the configured service may still be the legacy Modal
// endpoint, whose /batch_highlight contract uses {requests: [...]}. Keep
// highlights available until infra switches the URL to GCP Stage 1.
- if (res.status === 400 || res.status === 404) {
+ if (
+ opts.allowLegacyFallback !== false &&
+ (res.status === 400 || res.status === 404)
+ ) {
opts.logger.info("query highlights using legacy per-page fallback", {
canonicalLog: "search/highlights",
status: res.status,
@@ -121,10 +128,14 @@ export async function generateHighlightsBatch(
canonicalLog: "search/highlights",
requestedPages: pages.length,
returnedPages: results.size,
- pages: Array.from(results, ([id, result]) => ({
- id,
- highlights: result.highlights,
- })),
+ ...(opts.logPayload === false
+ ? {}
+ : {
+ pages: Array.from(results, ([id, result]) => ({
+ id,
+ highlights: result.highlights,
+ })),
+ }),
elapsedMs: Date.now() - start,
});
diff --git a/apps/api/src/search/highlights-shadow.test.ts b/apps/api/src/search/highlights-shadow.test.ts
new file mode 100644
index 000000000..b89acd37d
--- /dev/null
+++ b/apps/api/src/search/highlights-shadow.test.ts
@@ -0,0 +1,145 @@
+vi.mock("../config", () => ({
+ config: {
+ HIGHLIGHT_SHADOW_RATE: 1,
+ HIGHLIGHT_SHADOW_MAX_INFLIGHT: 1,
+ },
+}));
+
+vi.mock("./highlights", () => ({
+ highlightsEnvReady: vi.fn(() => true),
+ applySearchHighlights: vi.fn(),
+}));
+
+import { config } from "../config";
+import { applySearchHighlights, highlightsEnvReady } from "./highlights";
+import { createSearchHighlightsShadowRunner } from "./highlights-shadow";
+
+const logger = {
+ info: vi.fn(),
+ warn: vi.fn(),
+} as any;
+
+let runSearchHighlightsShadow = createSearchHighlightsShadowRunner();
+
+afterEach(() => {
+ vi.clearAllMocks();
+ config.HIGHLIGHT_SHADOW_RATE = 1;
+ config.HIGHLIGHT_SHADOW_MAX_INFLIGHT = 1;
+ vi.mocked(highlightsEnvReady).mockReturnValue(true);
+ runSearchHighlightsShadow = createSearchHighlightsShadowRunner();
+});
+
+describe("runSearchHighlightsShadow", () => {
+ it("runs without applying results and emits a content-free canonical log", async () => {
+ vi.mocked(applySearchHighlights).mockResolvedValue({
+ attempted: 10,
+ indexHits: 7,
+ replaced: 6,
+ succeeded: true,
+ });
+
+ expect(
+ runSearchHighlightsShadow({
+ response: {} as any,
+ query: "private query",
+ requestId: "request-1",
+ teamId: "team-1",
+ zeroDataRetention: false,
+ logger,
+ }),
+ ).toBe("started");
+ await new Promise(resolve => setImmediate(resolve));
+
+ expect(applySearchHighlights).toHaveBeenCalledWith(
+ {},
+ "private query",
+ expect.objectContaining({ silent: true }),
+ {
+ applyResults: false,
+ suppressSummaryLog: true,
+ suppressPayloadLog: true,
+ allowLegacyFallback: false,
+ },
+ );
+ expect(logger.info).toHaveBeenCalledWith(
+ "Search highlights shadow completed",
+ expect.objectContaining({
+ canonicalLog: "search/highlights-shadow",
+ outcome: "completed",
+ attempted: 10,
+ indexHits: 7,
+ wouldReplace: 6,
+ }),
+ );
+ const fields = logger.info.mock.calls[0][1];
+ expect(fields).not.toHaveProperty("query");
+ expect(fields).not.toHaveProperty("markdown");
+ expect(fields).not.toHaveProperty("highlights");
+ });
+
+ it("drops excess work instead of creating a backlog", async () => {
+ let finish!: (value: {
+ attempted: number;
+ indexHits: number;
+ replaced: number;
+ succeeded: boolean;
+ }) => void;
+ vi.mocked(applySearchHighlights).mockReturnValue(
+ new Promise(resolve => {
+ finish = resolve;
+ }),
+ );
+
+ const input = {
+ response: {} as any,
+ query: "query",
+ teamId: "team-1",
+ zeroDataRetention: false,
+ logger,
+ };
+ expect(
+ runSearchHighlightsShadow({ ...input, requestId: "request-1" }),
+ ).toBe("started");
+ expect(
+ runSearchHighlightsShadow({ ...input, requestId: "request-2" }),
+ ).toBe("dropped");
+ expect(logger.info).toHaveBeenCalledWith(
+ "Search highlights shadow dropped",
+ expect.objectContaining({
+ canonicalLog: "search/highlights-shadow",
+ outcome: "dropped",
+ reason: "max_inflight",
+ }),
+ );
+
+ finish({ attempted: 1, indexHits: 1, replaced: 1, succeeded: true });
+ await new Promise(resolve => setImmediate(resolve));
+ });
+
+ it("does not shadow ZDR, disabled, or unavailable requests", () => {
+ const input = {
+ response: {} as any,
+ query: "query",
+ requestId: "request-1",
+ teamId: "team-1",
+ logger,
+ };
+
+ expect(
+ runSearchHighlightsShadow({ ...input, zeroDataRetention: true }),
+ ).toBe("skipped");
+
+ config.HIGHLIGHT_SHADOW_RATE = 0;
+ expect(
+ runSearchHighlightsShadow({ ...input, zeroDataRetention: false }),
+ ).toBe("skipped");
+
+ config.HIGHLIGHT_SHADOW_RATE = 1;
+ vi.mocked(highlightsEnvReady).mockReturnValue(false);
+ expect(
+ runSearchHighlightsShadow({ ...input, zeroDataRetention: false }),
+ ).toBe("skipped");
+
+ expect(applySearchHighlights).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/search/highlights-shadow.ts b/apps/api/src/search/highlights-shadow.ts
new file mode 100644
index 000000000..2479ee91d
--- /dev/null
+++ b/apps/api/src/search/highlights-shadow.ts
@@ -0,0 +1,98 @@
+import { createLogger, type Logger } from "winston";
+import type { SearchV2Response } from "../lib/entities";
+import { config } from "../config";
+import { applySearchHighlights, highlightsEnvReady } from "./highlights";
+
+const CANONICAL_LOG = "search/highlights-shadow";
+const shadowLogger = createLogger({ silent: true });
+
+function sampled(requestId: string, rate: number): boolean {
+ if (rate <= 0) return false;
+ if (rate >= 1) return true;
+
+ let hash = 2166136261;
+ for (let i = 0; i < requestId.length; i++) {
+ hash ^= requestId.charCodeAt(i);
+ hash = Math.imul(hash, 16777619);
+ }
+ return (hash >>> 0) / 0x1_0000_0000 < rate;
+}
+
+export function createSearchHighlightsShadowRunner(): (options: {
+ response: SearchV2Response;
+ query: string;
+ requestId: string;
+ teamId: string;
+ zeroDataRetention: boolean;
+ logger: Logger;
+}) => "skipped" | "dropped" | "started" {
+ let inFlight = 0;
+
+ return options => {
+ if (
+ options.zeroDataRetention ||
+ !highlightsEnvReady() ||
+ !sampled(options.requestId, config.HIGHLIGHT_SHADOW_RATE)
+ ) {
+ return "skipped";
+ }
+
+ if (inFlight >= config.HIGHLIGHT_SHADOW_MAX_INFLIGHT) {
+ options.logger.info("Search highlights shadow dropped", {
+ canonicalLog: CANONICAL_LOG,
+ outcome: "dropped",
+ reason: "max_inflight",
+ requestId: options.requestId,
+ teamId: options.teamId,
+ inFlight,
+ maxInFlight: config.HIGHLIGHT_SHADOW_MAX_INFLIGHT,
+ sampleRate: config.HIGHLIGHT_SHADOW_RATE,
+ });
+ return "dropped";
+ }
+
+ inFlight++;
+ const startedAt = Date.now();
+ void applySearchHighlights(options.response, options.query, shadowLogger, {
+ applyResults: false,
+ suppressSummaryLog: true,
+ suppressPayloadLog: true,
+ allowLegacyFallback: false,
+ })
+ .then(result => {
+ options.logger.info("Search highlights shadow completed", {
+ canonicalLog: CANONICAL_LOG,
+ outcome: result.succeeded ? "completed" : "failed",
+ requestId: options.requestId,
+ teamId: options.teamId,
+ attempted: result.attempted,
+ indexHits: result.indexHits,
+ wouldReplace: result.replaced,
+ timeTakenMs: Date.now() - startedAt,
+ inFlight,
+ maxInFlight: config.HIGHLIGHT_SHADOW_MAX_INFLIGHT,
+ sampleRate: config.HIGHLIGHT_SHADOW_RATE,
+ });
+ })
+ .catch(error => {
+ options.logger.warn("Search highlights shadow failed", {
+ canonicalLog: CANONICAL_LOG,
+ outcome: "failed",
+ requestId: options.requestId,
+ teamId: options.teamId,
+ errorType: error instanceof Error ? error.name : "unknown",
+ timeTakenMs: Date.now() - startedAt,
+ inFlight,
+ maxInFlight: config.HIGHLIGHT_SHADOW_MAX_INFLIGHT,
+ sampleRate: config.HIGHLIGHT_SHADOW_RATE,
+ });
+ })
+ .finally(() => {
+ inFlight--;
+ });
+
+ return "started";
+ };
+}
+
+export const runSearchHighlightsShadow = createSearchHighlightsShadowRunner();
diff --git a/apps/api/src/search/highlights.test.ts b/apps/api/src/search/highlights.test.ts
index ccaa8791f..821a5017d 100644
--- a/apps/api/src/search/highlights.test.ts
+++ b/apps/api/src/search/highlights.test.ts
@@ -39,6 +39,7 @@ vi.mock("./highlight-model", () => ({
import { generateHighlightsBatch } from "./highlight-model";
import { config } from "../config";
+import { indexGetRecent5 } from "../db/rpc";
import { applySearchHighlights, highlightsEnvReady } from "./highlights";
const logger = {
@@ -98,7 +99,12 @@ describe("applySearchHighlights", () => {
);
expect(response.web[0].description).toBe("first highlight");
expect(response.news[0].snippet).toBe("second highlight");
- expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 2 });
+ expect(result).toEqual({
+ attempted: 2,
+ indexHits: 2,
+ replaced: 2,
+ succeeded: true,
+ });
});
it("preserves individual fallbacks for missing and empty batch pages", async () => {
@@ -118,7 +124,12 @@ describe("applySearchHighlights", () => {
"first fallback",
"second fallback",
]);
- expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 0 });
+ expect(result).toEqual({
+ attempted: 2,
+ indexHits: 2,
+ replaced: 0,
+ succeeded: true,
+ });
});
it("preserves every fallback when the batch request fails", async () => {
@@ -136,6 +147,71 @@ describe("applySearchHighlights", () => {
"first fallback",
"second fallback",
]);
- expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 0 });
+ expect(result).toEqual({
+ attempted: 2,
+ indexHits: 2,
+ replaced: 0,
+ succeeded: false,
+ });
+ });
+
+ it("runs without mutating the response or logging payloads in shadow mode", async () => {
+ vi.mocked(generateHighlightsBatch).mockResolvedValue(
+ new Map([["0", { highlights: [], markdown: "shadow highlight" }]]),
+ );
+ const response = {
+ web: [{ url: "https://first.test", description: "fallback" }],
+ } as any;
+
+ const result = await applySearchHighlights(response, "query", logger, {
+ applyResults: false,
+ suppressSummaryLog: true,
+ suppressPayloadLog: true,
+ allowLegacyFallback: false,
+ });
+
+ expect(response.web[0].description).toBe("fallback");
+ expect(result).toEqual({
+ attempted: 1,
+ indexHits: 1,
+ replaced: 1,
+ succeeded: true,
+ });
+ expect(generateHighlightsBatch).toHaveBeenCalledWith(
+ "query",
+ expect.any(Array),
+ {
+ logger,
+ logPayload: false,
+ allowLegacyFallback: false,
+ },
+ );
+ expect(logger.info).not.toHaveBeenCalledWith(
+ "Search highlights applied",
+ expect.anything(),
+ );
+ });
+
+ it("omits result URLs from shadow lookup failures", async () => {
+ vi.mocked(indexGetRecent5).mockRejectedValueOnce(
+ new Error("lookup failed"),
+ );
+ const response = {
+ web: [{ url: "https://private.test", description: "fallback" }],
+ } as any;
+
+ await applySearchHighlights(response, "query", logger, {
+ applyResults: false,
+ suppressSummaryLog: true,
+ suppressPayloadLog: true,
+ allowLegacyFallback: false,
+ });
+
+ expect(logger.warn).toHaveBeenCalledWith(
+ "highlights: index lookup failed",
+ {
+ error: "lookup failed",
+ },
+ );
});
});
diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts
index 62a8a0c3b..f04e9d6c6 100644
--- a/apps/api/src/search/highlights.ts
+++ b/apps/api/src/search/highlights.ts
@@ -44,6 +44,7 @@ const ERROR_COUNT_TO_REGISTER = 3;
async function getIndexedMarkdownForURL(
url: string,
logger: Logger,
+ logUrl = true,
): Promise<string | null> {
if (!useIndex) {
return null;
@@ -110,7 +111,7 @@ async function getIndexedMarkdownForURL(
} catch (error) {
logger.warn("highlights: index lookup failed", {
error: error instanceof Error ? error.message : String(error),
- url,
+ ...(logUrl ? { url } : {}),
});
return null;
}
@@ -129,8 +130,20 @@ export async function applySearchHighlights(
response: SearchV2Response,
query: string,
logger: Logger,
-): Promise<{ attempted: number; indexHits: number; replaced: number }> {
+ options: {
+ applyResults?: boolean;
+ suppressSummaryLog?: boolean;
+ suppressPayloadLog?: boolean;
+ allowLegacyFallback?: boolean;
+ } = {},
+): Promise<{
+ attempted: number;
+ indexHits: number;
+ replaced: number;
+ succeeded: boolean;
+}> {
const start = Date.now();
+ const applyResults = options.applyResults ?? true;
// Collect every result we could highlight, each with a setter for its snippet
// field: web results carry it in `description`, news results in `snippet`.
@@ -156,13 +169,15 @@ export async function applySearchHighlights(
const attempted = targets.length;
if (attempted === 0) {
- return { attempted, indexHits: 0, replaced: 0 };
+ return { attempted, indexHits: 0, replaced: 0, succeeded: true };
}
// Look up indexed markdown for every URL in parallel, keeping the markdown for
// each hit so we can send it to the highlight model service.
const markdowns = await Promise.all(
- targets.map(t => getIndexedMarkdownForURL(t.url, logger)),
+ targets.map(t =>
+ getIndexedMarkdownForURL(t.url, logger, !options.suppressPayloadLog),
+ ),
);
const hits: {
apply: (h: string) => void;
@@ -177,6 +192,7 @@ export async function applySearchHighlights(
// Send every hit in one request. IDs are local batch indexes, so a missing or
// empty response only falls back the corresponding provider snippet.
let replaced = 0;
+ let succeeded = true;
if (indexHits > 0) {
const results = await generateHighlightsBatch(
query,
@@ -184,25 +200,41 @@ export async function applySearchHighlights(
id: String(index),
markdown: hit.markdown,
})),
- { logger },
+ options.suppressPayloadLog || options.allowLegacyFallback === false
+ ? {
+ logger,
+ logPayload: !options.suppressPayloadLog,
+ allowLegacyFallback: options.allowLegacyFallback,
+ }
+ : { logger },
);
+ succeeded = results !== null;
if (results) {
hits.forEach((hit, index) => {
const snippet = results.get(String(index))?.markdown;
if (snippet?.trim()) {
- hit.apply(snippet);
+ if (applyResults) {
+ hit.apply(snippet);
+ }
replaced++;
}
});
}
}
- logger.info("Search highlights applied", {
+ if (!options.suppressSummaryLog) {
+ logger.info("Search highlights applied", {
+ attempted,
+ indexHits,
+ replaced,
+ timeTakenMs: Date.now() - start,
+ });
+ }
+
+ return {
attempted,
indexHits,
replaced,
- timeTakenMs: Date.now() - start,
- });
-
- return { attempted, indexHits, replaced };
+ succeeded,
+ };
}