Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): add staged highlights rollout (#4041)
Gergő Móriczf627d9faed01
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 9168f7f60..63c66a047 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -159,11 +159,13 @@ const configSchema = z.object({CLICKHOUSE_ANALYTICS_URL: z.string().optional(),CLICKHOUSE_ANALYTICS_DATABASE: z.string().optional(),- // Search highlights (beta): highlighter service base URL. TOKEN is optional+ // Search highlights: highlighter service base URL. TOKEN is optional// bearer auth for legacy/external services; the in-cluster service omits it.HIGHLIGHT_MODEL_URL: z.string().optional(),HIGHLIGHT_MODEL_TOKEN: z.string().optional(),- HIGHLIGHT_SHADOW_RATE: z.coerce.number().min(0).max(1).default(0),+ // Stable percentage of non-MCP/CLI cohorts whose generated highlights are+ // returned. The remaining eligible traffic still runs in shadow mode.+ HIGHLIGHT_ROLLOUT_PERCENT: z.coerce.number().min(0).max(100).default(0),// Exchange (routed data sources service)FIRE_EXCHANGE_URL: z.url().optional(),diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 58d13a271..8656a4371 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -236,6 +236,7 @@ export async function searchController({teamId: req.auth.team_id,origin: req.body.origin,+ integration: req.body.integration,apiKeyId: req.acuc?.api_key_id ?? null,flags: req.acuc?.flags ?? null,requestId: jobId,diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.tsindex fc4331605..2284d4d45 100644--- a/apps/api/src/controllers/v1/types.ts+++ b/apps/api/src/controllers/v1/types.ts@@ -1319,7 +1319,6 @@ export type TeamFlags = {// POST /v2/search/:jobId/feedback returns 403 TEAM_OPTED_OUT when true.searchFeedbackOptOut?: boolean;researchBeta?: boolean;- highlightsBeta?: boolean;enrichBeta?: boolean;professionalProfileCompanyDataBeta?: boolean;organizationDataSourceAccess?: Record<diff --git a/apps/api/src/controllers/v2/search-schema.test.ts b/apps/api/src/controllers/v2/search-schema.test.tsnew file mode 100644index 000000000..5b91617ca--- /dev/null+++ b/apps/api/src/controllers/v2/search-schema.test.ts@@ -0,0 +1,28 @@+import { describe, expect, it } from "vitest";+import { searchRequestSchema } from "./types";++describe("searchRequestSchema highlights", () => {+ it("preserves an omitted value for integration and rollout selection", () => {+ const request = searchRequestSchema.parse({ query: "firecrawl" });++ expect(request.highlights).toBeUndefined();+ });++ it("allows highlights to be enabled explicitly", () => {+ const request = searchRequestSchema.parse({+ query: "firecrawl",+ highlights: true,+ });++ expect(request.highlights).toBe(true);+ });++ it("allows highlights to be disabled explicitly", () => {+ const request = searchRequestSchema.parse({+ query: "firecrawl",+ highlights: false,+ });++ expect(request.highlights).toBe(false);+ });+});diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex a89d74e72..f52f45fd7 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -219,6 +219,7 @@ export async function searchController({teamId: req.auth.team_id,origin: req.body.origin,+ integration: req.body.integration,apiKeyId: req.acuc?.api_key_id ?? null,flags: req.acuc?.flags ?? null,requestId: agentRequestId ?? jobId,diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex 6ae3cece9..f5e15000f 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -1560,7 +1560,6 @@ export type TeamFlags = {debugBranding?: boolean;maxBrowserSessions?: number;researchBeta?: boolean;- highlightsBeta?: boolean;menuBeta?: boolean;enrichBeta?: boolean;professionalProfileCompanyDataBeta?: boolean;@@ -1960,10 +1959,10 @@ export const searchRequestSchema = ztimeout: z.int().positive().finite().prefault(60000),ignoreInvalidURLs: z.boolean().optional().prefault(false),asyncScraping: z.boolean().optional().prefault(false),- // Experimental: replace each result's snippet with query-relevant- // highlights pulled from our index (last 30 days), out-of-line from- // scrapeURL. Falls back to the provider snippet when the URL isn't indexed.- highlights: z.boolean().optional().prefault(false),+ // Replace each result's snippet with query-relevant highlights pulled from+ // our index. When omitted, the caller integration and rollout cohort decide+ // whether generated highlights are returned or only run in shadow mode.+ highlights: z.boolean().optional(),__searchPreviewToken: z.string().optional(),threatProtection: threatProtectionOverrideSchema.optional(),scrapeOptions: baseScrapeOptionsdiff --git a/apps/api/src/scraper/scrapeURL/transformers/menu.ts b/apps/api/src/scraper/scrapeURL/transformers/menu.tsindex a5e470612..598a1c44c 100644--- a/apps/api/src/scraper/scrapeURL/transformers/menu.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/menu.ts@@ -12,8 +12,8 @@ export async function fetchMenu(}// Beta gate: during beta the menu format is limited to teams with the menuBeta- // flag. Fail closed and silent (mirrors highlightsBeta) -- a team without the- // flag gets no menu field, no error, and no engine call.+ // flag. Fail closed and silent: a team without the flag gets no menu field,+ // no error, and no engine call.if (meta.internalOptions?.teamFlags?.menuBeta !== true) {meta.logger.info("menu format requested without menuBeta flag; skipping");return document;diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.tsindex 0fd58c123..5cab96b50 100644--- a/apps/api/src/search/execute.ts+++ b/apps/api/src/search/execute.ts@@ -13,13 +13,18 @@ import {mergeScrapedContent,calculateScrapeCredits,} from "./scrape";-import { applyIndexedSearchHighlights, highlightsEnvReady } from "./highlights";-import { runSearchHighlightsShadow } from "./highlights-shadow";+import {+ highlightsEnvReady,+ runIndexedSearchHighlights,+ searchHighlightsMode,+} from "./highlights";import { trackSearchResults, trackSearchRequest } from "../lib/tracking";import type { BillingMetadata } from "../services/billing/types";import type { ThreatProtectionPolicy } from "../lib/threat-protection/types";import { checkUrlsAgainstThreatPolicy } from "../lib/threat-protection/request";import { calculateThreatScanCredits } from "../lib/scrape-billing";+import { config } from "../config";+import { logger as rootLogger } from "../lib/logger";interface SearchOptions {query: string;@@ -42,6 +47,7 @@ interface SearchOptions {interface SearchContext {teamId: string;origin: string;+ integration?: string | null;apiKeyId: number | null;flags: TeamFlags;requestId: string;@@ -241,32 +247,47 @@ export async function executeSearch(}}- // Experimental highlights beta: replace provider snippets with index-backed- // highlights. Gated on (1) the request opting in, (2) the team's highlightsBeta- // flag, and (3) all required envs being present (index DB, GCS, model). Any- // gate failing => silently keep the provider snippets.+ // Every eligible request runs the same index-backed highlight path. MCP/CLI,+ // explicit opt-ins, and rollout-selected cohorts await and apply the result;+ // everyone else runs it in shadow without delaying or mutating the response.// Runs after scraping (mergeScrapedContent rebuilds the result objects, so// highlight mutations must come last to survive). Uses the user's original// query, not the domain-filtered upstream query.- const shouldApplyHighlights =- options.highlights &&- flags?.highlightsBeta === true &&- highlightsEnvReady();- if (shouldApplyHighlights) {- await applyIndexedSearchHighlights(+ if (zeroDataRetention !== true && !isZDR && highlightsEnvReady()) {+ const mode = searchHighlightsMode({+ requested: options.highlights,+ origin: context.origin,+ integration: context.integration,+ cohortKey:+ context.apiKeyId !== null+ ? `api-key:${context.apiKeyId}`+ : `team:${context.teamId}`,+ rolloutPercent: config.HIGHLIGHT_ROLLOUT_PERCENT,+ });+ const highlightRun = runIndexedSearchHighlights(searchResponse,query,logger,- context.requestId,+ {+ mode,+ requestId: context.requestId,+ teamId: context.teamId,+ },);- } else {- runSearchHighlightsShadow({- response: searchResponse,- query,- requestId: context.requestId,- teamId,- zeroDataRetention: zeroDataRetention === true || isZDR === true,- });++ if (mode === "apply") {+ await highlightRun;+ } else {+ void highlightRun.catch(error => {+ rootLogger.warn("Search highlights shadow failed", {+ canonicalLog: "search/highlights",+ mode,+ requestId: context.requestId,+ teamId: context.teamId,+ errorType: error instanceof Error ? error.name : "unknown",+ });+ });+ }}const scrapeFormats = scrapeOptions?.formatsdiff --git a/apps/api/src/search/highlights-shadow.test.ts b/apps/api/src/search/highlights-shadow.test.tsdeleted file mode 100644index eaef9684b..000000000--- a/apps/api/src/search/highlights-shadow.test.ts+++ /dev/null@@ -1,154 +0,0 @@-vi.mock("../config", () => ({- config: {- HIGHLIGHT_SHADOW_RATE: 1,- },-}));--vi.mock("./highlights", () => ({- highlightsEnvReady: vi.fn(() => true),- searchHighlightURLs: vi.fn(() => []),- runIndexedSearchHighlightsShadow: vi.fn(),-}));--import { config } from "../config";-import {- highlightsEnvReady,- runIndexedSearchHighlightsShadow,- searchHighlightURLs,-} from "./highlights";-import { createSearchHighlightsShadowRunner } from "./highlights-shadow";--const logger = {- info: vi.fn(),- warn: vi.fn(),-} as any;--let runSearchHighlightsShadow = createSearchHighlightsShadowRunner(logger);--afterEach(() => {- vi.clearAllMocks();- config.HIGHLIGHT_SHADOW_RATE = 1;- vi.mocked(highlightsEnvReady).mockReturnValue(true);- runSearchHighlightsShadow = createSearchHighlightsShadowRunner(logger);-});--describe("runSearchHighlightsShadow", () => {- it("runs without applying results and emits a content-free canonical log", async () => {- vi.mocked(runIndexedSearchHighlightsShadow).mockResolvedValue({- attempted: 10,- indexHits: 7,- replaced: 6,- succeeded: true,- });-- expect(- runSearchHighlightsShadow({- response: {} as any,- query: "private query",- requestId: "request-1",- teamId: "team-1",- zeroDataRetention: false,- }),- ).toBe("started");- await new Promise(resolve => setImmediate(resolve));-- expect(searchHighlightURLs).toHaveBeenCalledWith({});- expect(runIndexedSearchHighlightsShadow).toHaveBeenCalledWith(- [],- "private query",- expect.objectContaining({ silent: true }),- "request-1",- );- expect(logger.info).toHaveBeenCalledWith(- "Search highlights shadow completed",- expect.objectContaining({- canonicalLog: "search/highlights-shadow",- outcome: "completed",- attempted: 10,- indexHits: 7,- wouldReplace: 6,- }),- );- const fields = logger.info.mock.calls[0][1];- expect(fields).not.toHaveProperty("query");- expect(fields).not.toHaveProperty("markdown");- expect(fields).not.toHaveProperty("highlights");- });-- it("forwards concurrent shadow work without admission drops", async () => {- vi.mocked(runIndexedSearchHighlightsShadow).mockResolvedValue({- attempted: 1,- indexHits: 1,- replaced: 1,- succeeded: true,- });-- const input = {- response: {} as any,- query: "query",- teamId: "team-1",- zeroDataRetention: false,- };- expect(- runSearchHighlightsShadow({ ...input, requestId: "request-1" }),- ).toBe("started");- expect(- runSearchHighlightsShadow({ ...input, requestId: "request-2" }),- ).toBe("started");- await new Promise(resolve => setImmediate(resolve));- expect(runIndexedSearchHighlightsShadow).toHaveBeenCalledTimes(2);- });-- it("emits the content-free failure category", async () => {- vi.mocked(runIndexedSearchHighlightsShadow).mockResolvedValue({- attempted: 3,- indexHits: 1,- replaced: 0,- succeeded: false,- failureReason: "network",- });-- runSearchHighlightsShadow({- response: {} as any,- query: "private query",- requestId: "request-1",- teamId: "team-1",- zeroDataRetention: false,- });- await new Promise(resolve => setImmediate(resolve));-- expect(logger.info).toHaveBeenCalledWith(- "Search highlights shadow completed",- expect.objectContaining({- outcome: "failed",- failureReason: "network",- }),- );- });-- it("does not shadow ZDR, disabled, or unavailable requests", () => {- const input = {- response: {} as any,- query: "query",- requestId: "request-1",- teamId: "team-1",- };-- expect(- runSearchHighlightsShadow({ ...input, zeroDataRetention: true }),- ).toBe("skipped");-- config.HIGHLIGHT_SHADOW_RATE = 0;- expect(- runSearchHighlightsShadow({ ...input, zeroDataRetention: false }),- ).toBe("skipped");-- config.HIGHLIGHT_SHADOW_RATE = 1;- vi.mocked(highlightsEnvReady).mockReturnValue(false);- expect(- runSearchHighlightsShadow({ ...input, zeroDataRetention: false }),- ).toBe("skipped");-- expect(runIndexedSearchHighlightsShadow).not.toHaveBeenCalled();- });-});diff --git a/apps/api/src/search/highlights-shadow.ts b/apps/api/src/search/highlights-shadow.tsdeleted file mode 100644index aba928f0d..000000000--- a/apps/api/src/search/highlights-shadow.ts+++ /dev/null@@ -1,86 +0,0 @@-import { createLogger, type Logger } from "winston";-import type { SearchV2Response } from "../lib/entities";-import { config } from "../config";-import { logger as rootLogger } from "../lib/logger";-import {- highlightsEnvReady,- runIndexedSearchHighlightsShadow,- searchHighlightURLs,-} from "./highlights";--const CANONICAL_LOG = "search/highlights-shadow";-const shadowLogger = createLogger({ silent: true });--function sampled(requestId: string, rate: number): boolean {- if (rate <= 0) return false;- if (rate >= 1) return true;-- let hash = 2166136261;- for (let i = 0; i < requestId.length; i++) {- hash ^= requestId.charCodeAt(i);- hash = Math.imul(hash, 16777619);- }- return (hash >>> 0) / 0x1_0000_0000 < rate;-}--export function createSearchHighlightsShadowRunner(- canonicalLogger: Pick<Logger, "info" | "warn"> = rootLogger,-): (options: {- response: SearchV2Response;- query: string;- requestId: string;- teamId: string;- zeroDataRetention: boolean;-}) => "skipped" | "started" {- let inFlight = 0;-- return options => {- if (- options.zeroDataRetention ||- !highlightsEnvReady() ||- !sampled(options.requestId, config.HIGHLIGHT_SHADOW_RATE)- ) {- return "skipped";- }-- const urls = searchHighlightURLs(options.response);- const { query, requestId, teamId } = options;- inFlight++;- const startedAt = Date.now();- void runIndexedSearchHighlightsShadow(urls, query, shadowLogger, requestId)- .then(result => {- canonicalLogger.info("Search highlights shadow completed", {- canonicalLog: CANONICAL_LOG,- outcome: result.succeeded ? "completed" : "failed",- requestId,- teamId,- attempted: result.attempted,- indexHits: result.indexHits,- wouldReplace: result.replaced,- failureReason: result.failureReason,- timeTakenMs: Date.now() - startedAt,- inFlight,- sampleRate: config.HIGHLIGHT_SHADOW_RATE,- });- })- .catch(error => {- canonicalLogger.warn("Search highlights shadow failed", {- canonicalLog: CANONICAL_LOG,- outcome: "failed",- requestId,- teamId,- errorType: error instanceof Error ? error.name : "unknown",- timeTakenMs: Date.now() - startedAt,- inFlight,- sampleRate: config.HIGHLIGHT_SHADOW_RATE,- });- })- .finally(() => {- inFlight--;- });-- return "started";- };-}--export const runSearchHighlightsShadow = createSearchHighlightsShadowRunner();diff --git a/apps/api/src/search/highlights.test.ts b/apps/api/src/search/highlights.test.tsindex 2f6c11976..3ee497d3f 100644--- a/apps/api/src/search/highlights.test.ts+++ b/apps/api/src/search/highlights.test.ts@@ -3,6 +3,7 @@ vi.mock("../config", () => ({GCS_INDEX_BUCKET_NAME: "index-bucket",HIGHLIGHT_MODEL_URL: "https://highlight.test",HIGHLIGHT_MODEL_TOKEN: "secret-token",+ HIGHLIGHT_ROLLOUT_PERCENT: 0,},}));@@ -10,9 +11,6 @@ vi.mock("../services", () => ({useIndex: true,normalizeURLForIndex: vi.fn((url: string) => url),hashURL: vi.fn((url: string) => url),- getIndexFromGCS: vi.fn(async (key: string) => ({- html: `<main>${key}</main>`,- })),}));vi.mock("../db/rpc", () => ({@@ -25,31 +23,24 @@ vi.mock("../db/rpc", () => ({]),}));-vi.mock("../lib/html-to-markdown", () => ({- parseMarkdown: vi.fn(async (html: string) => `markdown:${html}`),-}));--vi.mock("../scraper/scrapeURL/lib/removeUnwantedElements", () => ({- htmlTransform: vi.fn(async (html: string) => html),+vi.mock("../lib/logger", () => ({+ logger: {+ info: vi.fn(),+ warn: vi.fn(),+ },}));vi.mock("./highlight-model", () => ({- generateHighlightsBatch: vi.fn(),generateHighlightsIndexedBatch: vi.fn(),}));-import {- generateHighlightsBatch,- generateHighlightsIndexedBatch,-} from "./highlight-model";+import { generateHighlightsIndexedBatch } from "./highlight-model";import { config } from "../config";-import { indexGetRecent5 } from "../db/rpc";+import { logger as rootLogger } from "../lib/logger";import {- applyIndexedSearchHighlights,- applySearchHighlights,highlightsEnvReady,- runIndexedSearchHighlightsShadow,- searchHighlightURLs,+ runIndexedSearchHighlights,+ searchHighlightsMode,} from "./highlights";const logger = {@@ -66,134 +57,7 @@ afterEach(() => {vi.clearAllMocks();});-describe("runIndexedSearchHighlightsShadow", () => {- it("resolves lightweight index references without loading page content", async () => {- vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(- new Map([["0", { highlights: [], markdown: "shadow highlight" }]]),- );- const response = {- web: [{ url: "https://first.test", description: "fallback" }],- news: [{ url: "https://second.test", snippet: "fallback" }],- } as any;- const urls = searchHighlightURLs(response);-- const result = await runIndexedSearchHighlightsShadow(- urls,- "query",- logger,- "request-1",- );-- expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith(- "query",- [- {- id: "0",- url: "https://first.test",- indexObject: "index:https://first.test.json",- },- {- id: "1",- url: "https://second.test",- indexObject: "index:https://second.test.json",- },- ],- {- logger,- logPayload: false,- requestId: "request-1",- timeoutMs: null,- onFailure: expect.any(Function),- },- );- expect(result).toEqual({- attempted: 2,- indexHits: 2,- replaced: 1,- succeeded: true,- });- });-});--describe("applyIndexedSearchHighlights", () => {- it("uses the shadow indexed request path and applies web and news responses by ID", async () => {- vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(- new Map([- ["0", { highlights: [], markdown: "first highlight" }],- ["1", { highlights: [], markdown: "second highlight" }],- ]),- );- const response = {- web: [{ url: "https://first.test", description: "first fallback" }],- news: [{ url: "https://second.test", snippet: "second fallback" }],- } as any;-- const result = await applyIndexedSearchHighlights(- response,- "query",- logger,- "request-1",- );-- expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith(- "query",- [- {- id: "0",- url: "https://first.test",- indexObject: "index:https://first.test.json",- },- {- id: "1",- url: "https://second.test",- indexObject: "index:https://second.test.json",- },- ],- {- logger,- logPayload: false,- requestId: "request-1",- timeoutMs: null,- onFailure: expect.any(Function),- },- );- expect(generateHighlightsBatch).not.toHaveBeenCalled();- expect(response.web[0].description).toBe("first highlight");- expect(response.news[0].snippet).toBe("second highlight");- expect(result).toEqual({- attempted: 2,- indexHits: 2,- replaced: 2,- succeeded: true,- });- });-- it("preserves provider snippets when the indexed Chain request fails", async () => {- vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(null);- const response = {- web: [{ url: "https://first.test", description: "first fallback" }],- news: [{ url: "https://second.test", snippet: "second fallback" }],- } as any;-- const result = await applyIndexedSearchHighlights(- response,- "query",- logger,- "request-1",- );-- expect(response.web[0].description).toBe("first fallback");- expect(response.news[0].snippet).toBe("second fallback");- expect(result).toEqual({- attempted: 2,- indexHits: 2,- replaced: 0,- succeeded: false,- });- });-});--describe("applySearchHighlights", () => {+describe("runIndexedSearchHighlights", () => {it("enables the in-cluster service without requiring a bearer token", () => {const token = config.HIGHLIGHT_MODEL_TOKEN;config.HIGHLIGHT_MODEL_TOKEN = undefined;@@ -205,8 +69,8 @@ describe("applySearchHighlights", () => {}});- it("sends indexed web and news results in one batch and applies responses by ID", async () => {- vi.mocked(generateHighlightsBatch).mockResolvedValue(+ it("uses indexed references and applies web and news responses", async () => {+ vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(new Map([["0", { highlights: [], markdown: "first highlight" }],["1", { highlights: [], markdown: "second highlight" }],@@ -217,23 +81,31 @@ describe("applySearchHighlights", () => {news: [{ url: "https://second.test", snippet: "second fallback" }],} as any;- const result = await applySearchHighlights(response, "query", logger);+ const result = await runIndexedSearchHighlights(response, "query", logger, {+ mode: "apply",+ requestId: "request-1",+ teamId: "team-1",+ });- expect(generateHighlightsBatch).toHaveBeenCalledTimes(1);- expect(generateHighlightsBatch).toHaveBeenCalledWith(+ expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith("query",[{id: "0",- markdown: "markdown:<main>index:https://first.test.json</main>",+ url: "https://first.test",+ indexObject: "index:https://first.test.json",},{id: "1",- markdown: "markdown:<main>index:https://second.test.json</main>",+ url: "https://second.test",+ indexObject: "index:https://second.test.json",},],{logger,+ logPayload: false,+ requestId: "request-1",+ timeoutMs: 3000,onFailure: expect.any(Function),},);@@ -245,69 +117,24 @@ describe("applySearchHighlights", () => {replaced: 2,succeeded: true,});- });-- it("preserves individual fallbacks for missing and empty batch pages", async () => {- vi.mocked(generateHighlightsBatch).mockResolvedValue(- new Map([["0", { highlights: [], markdown: " " }]]),+ expect(logger.info).toHaveBeenCalledWith(+ "Search highlights completed",+ expect.objectContaining({ mode: "apply", applied: 2 }),);- const response = {- web: [- { url: "https://first.test", description: "first fallback" },- { url: "https://second.test", description: "second fallback" },- ],- } as any;-- const result = await applySearchHighlights(response, "query", logger);-- expect(response.web.map((item: any) => item.description)).toEqual([- "first fallback",- "second fallback",- ]);- expect(result).toEqual({- attempted: 2,- indexHits: 2,- replaced: 0,- succeeded: true,- });});- it("preserves every fallback when the batch request fails", async () => {- vi.mocked(generateHighlightsBatch).mockResolvedValue(null);- const response = {- web: [- { url: "https://first.test", description: "first fallback" },- { url: "https://second.test", description: "second fallback" },- ],- } as any;-- const result = await applySearchHighlights(response, "query", logger);-- expect(response.web.map((item: any) => item.description)).toEqual([- "first fallback",- "second fallback",- ]);- expect(result).toEqual({- attempted: 2,- indexHits: 2,- replaced: 0,- succeeded: false,- });- });-- it("runs without mutating the response or logging payloads in shadow mode", async () => {- vi.mocked(generateHighlightsBatch).mockResolvedValue(+ it("runs the same indexed path without mutating the response in shadow mode", async () => {+ vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(new Map([["0", { highlights: [], markdown: "shadow highlight" }]]),);const response = {web: [{ url: "https://first.test", description: "fallback" }],} as any;- const result = await applySearchHighlights(response, "query", logger, {- applyResults: false,- suppressSummaryLog: true,- suppressPayloadLog: true,- allowLegacyFallback: false,+ const result = await runIndexedSearchHighlights(response, "query", logger, {+ mode: "shadow",+ requestId: "request-1",+ teamId: "team-1",});expect(response.web[0].description).toBe("fallback");@@ -317,42 +144,71 @@ describe("applySearchHighlights", () => {replaced: 1,succeeded: true,});- expect(generateHighlightsBatch).toHaveBeenCalledWith(+ expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith("query",expect.any(Array),- {- logger,+ expect.objectContaining({+ logger: expect.objectContaining({ silent: true }),logPayload: false,- allowLegacyFallback: false,- onFailure: expect.any(Function),- },+ }),);- expect(logger.info).not.toHaveBeenCalledWith(- "Search highlights applied",- expect.anything(),+ expect(rootLogger.info).toHaveBeenCalledWith(+ "Search highlights completed",+ expect.objectContaining({ mode: "shadow", wouldApply: 1 }),);+ expect(logger.info).not.toHaveBeenCalled();});- it("omits result URLs from shadow lookup failures", async () => {- vi.mocked(indexGetRecent5).mockRejectedValueOnce(- new Error("lookup failed"),- );+ it("preserves provider snippets when the indexed request fails", async () => {+ vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(null);const response = {- web: [{ url: "https://private.test", description: "fallback" }],+ web: [{ url: "https://first.test", description: "first fallback" }],} as any;- await applySearchHighlights(response, "query", logger, {- applyResults: false,- suppressSummaryLog: true,- suppressPayloadLog: true,- allowLegacyFallback: false,+ const result = await runIndexedSearchHighlights(response, "query", logger, {+ mode: "apply",+ requestId: "request-1",+ teamId: "team-1",});- expect(logger.warn).toHaveBeenCalledWith(- "highlights: index lookup failed",- {- error: "lookup failed",- },+ expect(response.web[0].description).toBe("first fallback");+ expect(result).toEqual({+ attempted: 1,+ indexHits: 1,+ replaced: 0,+ succeeded: false,+ });+ });+});++describe("searchHighlightsMode", () => {+ const base = {+ cohortKey: "api-key:123",+ rolloutPercent: 0,+ };++ it("applies omitted highlights for MCP and CLI callers", () => {+ expect(searchHighlightsMode({ ...base, origin: "mcp" })).toBe("apply");+ expect(searchHighlightsMode({ ...base, origin: "mcp-fastmcp" })).toBe(+ "apply",+ );+ expect(searchHighlightsMode({ ...base, integration: "cli" })).toBe("apply");+ });++ it("honors explicit response selection before caller defaults", () => {+ expect(searchHighlightsMode({ ...base, requested: true })).toBe("apply");+ expect(+ searchHighlightsMode({ ...base, requested: false, origin: "mcp" }),+ ).toBe("shadow");+ });++ it("uses the rollout percentage for other callers", () => {+ expect(searchHighlightsMode({ ...base, rolloutPercent: 0 })).toBe("shadow");+ expect(searchHighlightsMode({ ...base, rolloutPercent: 100 })).toBe(+ "apply",);++ const first = searchHighlightsMode({ ...base, rolloutPercent: 50 });+ expect(searchHighlightsMode({ ...base, rolloutPercent: 50 })).toBe(first);});});diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.tsindex a8448c7c1..1dd158c84 100644--- a/apps/api/src/search/highlights.ts+++ b/apps/api/src/search/highlights.ts@@ -1,28 +1,24 @@-import type { Logger } from "winston";+import { createLogger, type Logger } from "winston";import { SearchV2Response } from "../lib/entities";-import {- normalizeURLForIndex,- hashURL,- getIndexFromGCS,- useIndex,-} from "../services";+import { normalizeURLForIndex, hashURL, useIndex } from "../services";import { indexGetRecent5 } from "../db/rpc";-import { parseMarkdown } from "../lib/html-to-markdown";-import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";-import type { ScrapeOptions } from "../controllers/v2/types";import {- generateHighlightsBatch,generateHighlightsIndexedBatch,type HighlightIndexedPage,} from "./highlight-model";import type { HighlightFailureReason } from "./highlight-model";import { config } from "../config";+import { logger as rootLogger } from "../lib/logger";// How far back into the index we're willing to reach for highlight source text.const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days+const HIGHLIGHTS_API_TIMEOUT_MS = 3000;+const shadowLogger = createLogger({ silent: true });++type SearchHighlightsMode = "apply" | "shadow";/**- * Whether the deployment has every dependency the highlights beta needs: the+ * Whether the deployment has every dependency indexed highlights need: the* index DB (to find cached content), the GCS index bucket (to fetch it), and the* highlight model service URL (to score it). Missing any => silently skip.*/@@ -32,66 +28,43 @@ export function highlightsEnvReady(): boolean {);}-// Mirrors scrapeURLWithIndex: prefer the newest 2xx entry unless it sits behind-// this many more-recent error entries, in which case we surface the newest one.-const ERROR_COUNT_TO_REGISTER = 3;--// This whole module runs out-of-line from scrapeURL on purpose: it reads-// already-indexed content directly from the index DB + GCS instead of routing-// through the scrape engine. That keeps highlight generation off the critical-// scrape path and lets us experiment with latency freely.+function sampled(cohortKey: string, percent: number): boolean {+ if (percent <= 0) return false;+ if (percent >= 100) return true;-/**- * Fetch the most recent indexed markdown for a URL within the last 30 days.- * Returns null when the URL isn't in the index (or the lookup fails) so callers- * can fall back to the provider snippet.- */-async function getIndexedMarkdownForURL(- url: string,- logger: Logger,- logUrl = true,-): Promise<string | null> {- const indexRef = await getIndexObjectForURL(url, logger, logUrl);- if (!indexRef) {- return null;+ let hash = 2166136261;+ for (let i = 0; i < cohortKey.length; i++) {+ hash ^= cohortKey.charCodeAt(i);+ hash = Math.imul(hash, 16777619);}+ return ((hash >>> 0) / 0x1_0000_0000) * 100 < percent;+}- try {- const doc = await getIndexFromGCS(- indexRef.name,- logger.child({ module: "search/highlights", method: "getIndexFromGCS" }),- { indexCreatedAt: indexRef.createdAt },- );- if (!doc || !doc.html) {- return null;- }-- // Skip raw base64 PDFs — they aren't useful as highlight source text.- if (typeof doc.html === "string" && doc.html.startsWith("JVBERi")) {- return null;- }-- // The index stores rawHtml, so we must run the same cleaning the scrape- // pipeline does (strip <style>/<script>/nav, extract main content) before- // converting to markdown — otherwise CSS/JS leaks in and pollutes the- // highlight source text.- const cleanedHtml = await htmlTransform(doc.html, url, {- onlyMainContent: true,- includeTags: [],- excludeTags: [],- } as unknown as ScrapeOptions);-- const markdown = await parseMarkdown(cleanedHtml, { logger });- return markdown && markdown.trim() !== "" ? markdown : null;- } catch (error) {- logger.warn("highlights: index content load failed", {- error: error instanceof Error ? error.message : String(error),- ...(logUrl ? { url } : {}),- });- return null;+export function searchHighlightsMode(options: {+ requested?: boolean;+ origin?: string | null;+ integration?: string | null;+ cohortKey: string;+ rolloutPercent: number;+}): SearchHighlightsMode {+ if (options.requested === true) return "apply";+ if (options.requested === false) return "shadow";++ const origin = (options.origin ?? "").toLowerCase();+ const integration = (options.integration ?? "").toLowerCase();+ if (integration === "cli" || origin === "cli" || origin.startsWith("mcp")) {+ return "apply";}++ return sampled(options.cohortKey, options.rolloutPercent)+ ? "apply"+ : "shadow";}+// Mirrors scrapeURLWithIndex: prefer the newest 2xx entry unless it sits behind+// this many more-recent error entries, in which case we surface the newest one.+const ERROR_COUNT_TO_REGISTER = 3;+async function getIndexObjectForURL(url: string,logger: Logger,@@ -176,15 +149,12 @@ function indexedSearchHighlightTargets(return targets;}-export function searchHighlightURLs(response: SearchV2Response): string[] {- return indexedSearchHighlightTargets(response).map(target => target.url);-}--async function runIndexedSearchHighlights(+async function generateIndexedSearchHighlights(targets: IndexedSearchHighlightTarget[],query: string,logger: Logger,requestId: string,+ applyResults: boolean,): Promise<{attempted: number;indexHits: number;@@ -211,7 +181,7 @@ async function runIndexedSearchHighlights(logger,logPayload: false,requestId,- timeoutMs: null,+ timeoutMs: HIGHLIGHTS_API_TIMEOUT_MS,onFailure: reason => {failureReason = reason;},@@ -221,7 +191,9 @@ async function runIndexedSearchHighlights(for (const page of pages) {const highlight = results.get(page.id)?.markdown;if (!highlight?.trim()) continue;- targets[Number(page.id)].apply?.(highlight);+ if (applyResults) {+ targets[Number(page.id)].apply?.(highlight);+ }replaced++;}}@@ -235,174 +207,38 @@ async function runIndexedSearchHighlights(};}-export async function applyIndexedSearchHighlights(+export async function runIndexedSearchHighlights(response: SearchV2Response,query: string,logger: Logger,- requestId: string,-): ReturnType<typeof runIndexedSearchHighlights> {+ options: {+ mode: SearchHighlightsMode;+ requestId: string;+ teamId: string;+ },+): ReturnType<typeof generateIndexedSearchHighlights> {const start = Date.now();- const result = await runIndexedSearchHighlights(+ const result = await generateIndexedSearchHighlights(indexedSearchHighlightTargets(response),query,- logger,- requestId,+ options.mode === "shadow" ? shadowLogger : logger,+ options.requestId,+ options.mode === "apply",);- logger.info("Search highlights applied", {+ const summaryLogger = options.mode === "shadow" ? rootLogger : logger;+ summaryLogger.info("Search highlights completed", {+ canonicalLog: "search/highlights",+ mode: options.mode,+ outcome: result.succeeded ? "completed" : "failed",+ requestId: options.requestId,+ teamId: options.teamId,attempted: result.attempted,indexHits: result.indexHits,- replaced: result.replaced,+ ...(options.mode === "apply"+ ? { applied: result.replaced }+ : { wouldApply: result.replaced }),+ failureReason: result.failureReason,timeTakenMs: Date.now() - start,});return result;}--export function runIndexedSearchHighlightsShadow(- urls: string[],- query: string,- logger: Logger,- requestId: string,-): ReturnType<typeof runIndexedSearchHighlights> {- return runIndexedSearchHighlights(- urls.map(url => ({ url })),- query,- logger,- requestId,- );-}--/**- * For each search result: look up the URL in our index (last 30 days), and if- * present, replace the provider snippet with query-relevant highlights generated- * from the indexed content. Index lookups run in parallel; each hit's full- * markdown pages are sent to the highlight model service in one batch, which- * returns each page's selected highlights reassembled into a single markdown- * document. Mutates `response` in place. Results not in the index keep their- * original snippet.- */-export async function applySearchHighlights(- response: SearchV2Response,- query: string,- logger: Logger,- options: {- applyResults?: boolean;- suppressSummaryLog?: boolean;- suppressPayloadLog?: boolean;- allowLegacyFallback?: boolean;- requestId?: string;- } = {},-): Promise<{- attempted: number;- indexHits: number;- replaced: number;- succeeded: boolean;- failureReason?: HighlightFailureReason;-}> {- const start = Date.now();- const applyResults = options.applyResults ?? true;-- // Collect every result we could highlight, each with a setter for its snippet- // field: web results carry it in `description`, news results in `snippet`.- const targets: { url: string; apply: (h: string) => void }[] = [];- for (const result of response.web ?? []) {- if (!result.url) continue;- targets.push({- url: result.url,- apply: h => {- result.description = h;- },- });- }- for (const result of response.news ?? []) {- if (!result.url) continue;- targets.push({- url: result.url,- apply: h => {- result.snippet = h;- },- });- }-- const attempted = targets.length;- if (attempted === 0) {- return { attempted, indexHits: 0, replaced: 0, succeeded: true };- }-- // Look up indexed markdown for every URL in parallel, keeping the markdown for- // each hit so we can send it to the highlight model service.- const markdowns = await Promise.all(- targets.map(t =>- getIndexedMarkdownForURL(t.url, logger, !options.suppressPayloadLog),- ),- );- const hits: {- apply: (h: string) => void;- markdown: string;- }[] = [];- markdowns.forEach((markdown, i) => {- if (!markdown) return;- hits.push({ apply: targets[i].apply, markdown });- });- const indexHits = hits.length;-- // Send every hit in one request. IDs are local batch indexes, so a missing or- // empty response only falls back the corresponding provider snippet.- let replaced = 0;- let succeeded = true;- let failureReason: HighlightFailureReason | undefined;- if (indexHits > 0) {- const results = await generateHighlightsBatch(- query,- hits.map((hit, index) => ({- id: String(index),- markdown: hit.markdown,- })),- options.suppressPayloadLog || options.allowLegacyFallback === false- ? {- logger,- logPayload: !options.suppressPayloadLog,- allowLegacyFallback: options.allowLegacyFallback,- ...(options.requestId ? { requestId: options.requestId } : {}),- onFailure: reason => {- failureReason = reason;- },- }- : {- logger,- ...(options.requestId ? { requestId: options.requestId } : {}),- onFailure: reason => {- failureReason = reason;- },- },- );- succeeded = results !== null;- if (results) {- hits.forEach((hit, index) => {- const snippet = results.get(String(index))?.markdown;- if (snippet?.trim()) {- if (applyResults) {- hit.apply(snippet);- }- replaced++;- }- });- }- }-- if (!options.suppressSummaryLog) {- logger.info("Search highlights applied", {- attempted,- indexHits,- replaced,- timeTakenMs: Date.now() - start,- });- }-- return {- attempted,- indexHits,- replaced,- succeeded,- ...(failureReason ? { failureReason } : {}),- };-}