Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(search): add staged highlights rollout (#4041)

Gergő Móriczf627d9faed01
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 9168f7f60..63c66a047 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -159,11 +159,13 @@ const configSchema = z.object({
CLICKHOUSE_ANALYTICS_URL: z.string().optional(),
CLICKHOUSE_ANALYTICS_DATABASE: z.string().optional(),
- // Search highlights (beta): highlighter service base URL. TOKEN is optional
+ // Search highlights: highlighter service base URL. TOKEN is optional
// bearer auth for legacy/external services; the in-cluster service omits it.
HIGHLIGHT_MODEL_URL: z.string().optional(),
HIGHLIGHT_MODEL_TOKEN: z.string().optional(),
- HIGHLIGHT_SHADOW_RATE: z.coerce.number().min(0).max(1).default(0),
+ // Stable percentage of non-MCP/CLI cohorts whose generated highlights are
+ // returned. The remaining eligible traffic still runs in shadow mode.
+ HIGHLIGHT_ROLLOUT_PERCENT: z.coerce.number().min(0).max(100).default(0),
// Exchange (routed data sources service)
FIRE_EXCHANGE_URL: z.url().optional(),
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 58d13a271..8656a4371 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -236,6 +236,7 @@ export async function searchController(
{
teamId: req.auth.team_id,
origin: req.body.origin,
+ integration: req.body.integration,
apiKeyId: req.acuc?.api_key_id ?? null,
flags: req.acuc?.flags ?? null,
requestId: jobId,
diff --git a/apps/api/src/controllers/v1/types.ts b/apps/api/src/controllers/v1/types.ts
index fc4331605..2284d4d45 100644
--- a/apps/api/src/controllers/v1/types.ts
+++ b/apps/api/src/controllers/v1/types.ts
@@ -1319,7 +1319,6 @@ export type TeamFlags = {
// POST /v2/search/:jobId/feedback returns 403 TEAM_OPTED_OUT when true.
searchFeedbackOptOut?: boolean;
researchBeta?: boolean;
- highlightsBeta?: boolean;
enrichBeta?: boolean;
professionalProfileCompanyDataBeta?: boolean;
organizationDataSourceAccess?: Record<
diff --git a/apps/api/src/controllers/v2/search-schema.test.ts b/apps/api/src/controllers/v2/search-schema.test.ts
new file mode 100644
index 000000000..5b91617ca
--- /dev/null
+++ b/apps/api/src/controllers/v2/search-schema.test.ts
@@ -0,0 +1,28 @@
+import { describe, expect, it } from "vitest";
+import { searchRequestSchema } from "./types";
+
+describe("searchRequestSchema highlights", () => {
+ it("preserves an omitted value for integration and rollout selection", () => {
+ const request = searchRequestSchema.parse({ query: "firecrawl" });
+
+ expect(request.highlights).toBeUndefined();
+ });
+
+ it("allows highlights to be enabled explicitly", () => {
+ const request = searchRequestSchema.parse({
+ query: "firecrawl",
+ highlights: true,
+ });
+
+ expect(request.highlights).toBe(true);
+ });
+
+ it("allows highlights to be disabled explicitly", () => {
+ const request = searchRequestSchema.parse({
+ query: "firecrawl",
+ highlights: false,
+ });
+
+ expect(request.highlights).toBe(false);
+ });
+});
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index a89d74e72..f52f45fd7 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -219,6 +219,7 @@ export async function searchController(
{
teamId: req.auth.team_id,
origin: req.body.origin,
+ integration: req.body.integration,
apiKeyId: req.acuc?.api_key_id ?? null,
flags: req.acuc?.flags ?? null,
requestId: agentRequestId ?? jobId,
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index 6ae3cece9..f5e15000f 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -1560,7 +1560,6 @@ export type TeamFlags = {
debugBranding?: boolean;
maxBrowserSessions?: number;
researchBeta?: boolean;
- highlightsBeta?: boolean;
menuBeta?: boolean;
enrichBeta?: boolean;
professionalProfileCompanyDataBeta?: boolean;
@@ -1960,10 +1959,10 @@ export const searchRequestSchema = z
timeout: z.int().positive().finite().prefault(60000),
ignoreInvalidURLs: z.boolean().optional().prefault(false),
asyncScraping: z.boolean().optional().prefault(false),
- // Experimental: replace each result's snippet with query-relevant
- // highlights pulled from our index (last 30 days), out-of-line from
- // scrapeURL. Falls back to the provider snippet when the URL isn't indexed.
- highlights: z.boolean().optional().prefault(false),
+ // Replace each result's snippet with query-relevant highlights pulled from
+ // our index. When omitted, the caller integration and rollout cohort decide
+ // whether generated highlights are returned or only run in shadow mode.
+ highlights: z.boolean().optional(),
__searchPreviewToken: z.string().optional(),
threatProtection: threatProtectionOverrideSchema.optional(),
scrapeOptions: baseScrapeOptions
diff --git a/apps/api/src/scraper/scrapeURL/transformers/menu.ts b/apps/api/src/scraper/scrapeURL/transformers/menu.ts
index a5e470612..598a1c44c 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/menu.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/menu.ts
@@ -12,8 +12,8 @@ export async function fetchMenu(
}
// Beta gate: during beta the menu format is limited to teams with the menuBeta
- // flag. Fail closed and silent (mirrors highlightsBeta) -- a team without the
- // flag gets no menu field, no error, and no engine call.
+ // flag. Fail closed and silent: a team without the flag gets no menu field,
+ // no error, and no engine call.
if (meta.internalOptions?.teamFlags?.menuBeta !== true) {
meta.logger.info("menu format requested without menuBeta flag; skipping");
return document;
diff --git a/apps/api/src/search/execute.ts b/apps/api/src/search/execute.ts
index 0fd58c123..5cab96b50 100644
--- a/apps/api/src/search/execute.ts
+++ b/apps/api/src/search/execute.ts
@@ -13,13 +13,18 @@ import {
mergeScrapedContent,
calculateScrapeCredits,
} from "./scrape";
-import { applyIndexedSearchHighlights, highlightsEnvReady } from "./highlights";
-import { runSearchHighlightsShadow } from "./highlights-shadow";
+import {
+ highlightsEnvReady,
+ runIndexedSearchHighlights,
+ searchHighlightsMode,
+} from "./highlights";
import { trackSearchResults, trackSearchRequest } from "../lib/tracking";
import type { BillingMetadata } from "../services/billing/types";
import type { ThreatProtectionPolicy } from "../lib/threat-protection/types";
import { checkUrlsAgainstThreatPolicy } from "../lib/threat-protection/request";
import { calculateThreatScanCredits } from "../lib/scrape-billing";
+import { config } from "../config";
+import { logger as rootLogger } from "../lib/logger";
interface SearchOptions {
query: string;
@@ -42,6 +47,7 @@ interface SearchOptions {
interface SearchContext {
teamId: string;
origin: string;
+ integration?: string | null;
apiKeyId: number | null;
flags: TeamFlags;
requestId: string;
@@ -241,32 +247,47 @@ export async function executeSearch(
}
}
- // Experimental highlights beta: replace provider snippets with index-backed
- // highlights. Gated on (1) the request opting in, (2) the team's highlightsBeta
- // flag, and (3) all required envs being present (index DB, GCS, model). Any
- // gate failing => silently keep the provider snippets.
+ // Every eligible request runs the same index-backed highlight path. MCP/CLI,
+ // explicit opt-ins, and rollout-selected cohorts await and apply the result;
+ // everyone else runs it in shadow without delaying or mutating the response.
// Runs after scraping (mergeScrapedContent rebuilds the result objects, so
// highlight mutations must come last to survive). Uses the user's original
// query, not the domain-filtered upstream query.
- const shouldApplyHighlights =
- options.highlights &&
- flags?.highlightsBeta === true &&
- highlightsEnvReady();
- if (shouldApplyHighlights) {
- await applyIndexedSearchHighlights(
+ if (zeroDataRetention !== true && !isZDR && highlightsEnvReady()) {
+ const mode = searchHighlightsMode({
+ requested: options.highlights,
+ origin: context.origin,
+ integration: context.integration,
+ cohortKey:
+ context.apiKeyId !== null
+ ? `api-key:${context.apiKeyId}`
+ : `team:${context.teamId}`,
+ rolloutPercent: config.HIGHLIGHT_ROLLOUT_PERCENT,
+ });
+ const highlightRun = runIndexedSearchHighlights(
searchResponse,
query,
logger,
- context.requestId,
+ {
+ mode,
+ requestId: context.requestId,
+ teamId: context.teamId,
+ },
);
- } else {
- runSearchHighlightsShadow({
- response: searchResponse,
- query,
- requestId: context.requestId,
- teamId,
- zeroDataRetention: zeroDataRetention === true || isZDR === true,
- });
+
+ if (mode === "apply") {
+ await highlightRun;
+ } else {
+ void highlightRun.catch(error => {
+ rootLogger.warn("Search highlights shadow failed", {
+ canonicalLog: "search/highlights",
+ mode,
+ requestId: context.requestId,
+ teamId: context.teamId,
+ errorType: error instanceof Error ? error.name : "unknown",
+ });
+ });
+ }
}
const scrapeFormats = scrapeOptions?.formats
diff --git a/apps/api/src/search/highlights-shadow.test.ts b/apps/api/src/search/highlights-shadow.test.ts
deleted file mode 100644
index eaef9684b..000000000
--- a/apps/api/src/search/highlights-shadow.test.ts
+++ /dev/null
@@ -1,154 +0,0 @@
-vi.mock("../config", () => ({
- config: {
- HIGHLIGHT_SHADOW_RATE: 1,
- },
-}));
-
-vi.mock("./highlights", () => ({
- highlightsEnvReady: vi.fn(() => true),
- searchHighlightURLs: vi.fn(() => []),
- runIndexedSearchHighlightsShadow: vi.fn(),
-}));
-
-import { config } from "../config";
-import {
- highlightsEnvReady,
- runIndexedSearchHighlightsShadow,
- searchHighlightURLs,
-} from "./highlights";
-import { createSearchHighlightsShadowRunner } from "./highlights-shadow";
-
-const logger = {
- info: vi.fn(),
- warn: vi.fn(),
-} as any;
-
-let runSearchHighlightsShadow = createSearchHighlightsShadowRunner(logger);
-
-afterEach(() => {
- vi.clearAllMocks();
- config.HIGHLIGHT_SHADOW_RATE = 1;
- vi.mocked(highlightsEnvReady).mockReturnValue(true);
- runSearchHighlightsShadow = createSearchHighlightsShadowRunner(logger);
-});
-
-describe("runSearchHighlightsShadow", () => {
- it("runs without applying results and emits a content-free canonical log", async () => {
- vi.mocked(runIndexedSearchHighlightsShadow).mockResolvedValue({
- attempted: 10,
- indexHits: 7,
- replaced: 6,
- succeeded: true,
- });
-
- expect(
- runSearchHighlightsShadow({
- response: {} as any,
- query: "private query",
- requestId: "request-1",
- teamId: "team-1",
- zeroDataRetention: false,
- }),
- ).toBe("started");
- await new Promise(resolve => setImmediate(resolve));
-
- expect(searchHighlightURLs).toHaveBeenCalledWith({});
- expect(runIndexedSearchHighlightsShadow).toHaveBeenCalledWith(
- [],
- "private query",
- expect.objectContaining({ silent: true }),
- "request-1",
- );
- expect(logger.info).toHaveBeenCalledWith(
- "Search highlights shadow completed",
- expect.objectContaining({
- canonicalLog: "search/highlights-shadow",
- outcome: "completed",
- attempted: 10,
- indexHits: 7,
- wouldReplace: 6,
- }),
- );
- const fields = logger.info.mock.calls[0][1];
- expect(fields).not.toHaveProperty("query");
- expect(fields).not.toHaveProperty("markdown");
- expect(fields).not.toHaveProperty("highlights");
- });
-
- it("forwards concurrent shadow work without admission drops", async () => {
- vi.mocked(runIndexedSearchHighlightsShadow).mockResolvedValue({
- attempted: 1,
- indexHits: 1,
- replaced: 1,
- succeeded: true,
- });
-
- const input = {
- response: {} as any,
- query: "query",
- teamId: "team-1",
- zeroDataRetention: false,
- };
- expect(
- runSearchHighlightsShadow({ ...input, requestId: "request-1" }),
- ).toBe("started");
- expect(
- runSearchHighlightsShadow({ ...input, requestId: "request-2" }),
- ).toBe("started");
- await new Promise(resolve => setImmediate(resolve));
- expect(runIndexedSearchHighlightsShadow).toHaveBeenCalledTimes(2);
- });
-
- it("emits the content-free failure category", async () => {
- vi.mocked(runIndexedSearchHighlightsShadow).mockResolvedValue({
- attempted: 3,
- indexHits: 1,
- replaced: 0,
- succeeded: false,
- failureReason: "network",
- });
-
- runSearchHighlightsShadow({
- response: {} as any,
- query: "private query",
- requestId: "request-1",
- teamId: "team-1",
- zeroDataRetention: false,
- });
- await new Promise(resolve => setImmediate(resolve));
-
- expect(logger.info).toHaveBeenCalledWith(
- "Search highlights shadow completed",
- expect.objectContaining({
- outcome: "failed",
- failureReason: "network",
- }),
- );
- });
-
- it("does not shadow ZDR, disabled, or unavailable requests", () => {
- const input = {
- response: {} as any,
- query: "query",
- requestId: "request-1",
- teamId: "team-1",
- };
-
- expect(
- runSearchHighlightsShadow({ ...input, zeroDataRetention: true }),
- ).toBe("skipped");
-
- config.HIGHLIGHT_SHADOW_RATE = 0;
- expect(
- runSearchHighlightsShadow({ ...input, zeroDataRetention: false }),
- ).toBe("skipped");
-
- config.HIGHLIGHT_SHADOW_RATE = 1;
- vi.mocked(highlightsEnvReady).mockReturnValue(false);
- expect(
- runSearchHighlightsShadow({ ...input, zeroDataRetention: false }),
- ).toBe("skipped");
-
- expect(runIndexedSearchHighlightsShadow).not.toHaveBeenCalled();
- });
-});
diff --git a/apps/api/src/search/highlights-shadow.ts b/apps/api/src/search/highlights-shadow.ts
deleted file mode 100644
index aba928f0d..000000000
--- a/apps/api/src/search/highlights-shadow.ts
+++ /dev/null
@@ -1,86 +0,0 @@
-import { createLogger, type Logger } from "winston";
-import type { SearchV2Response } from "../lib/entities";
-import { config } from "../config";
-import { logger as rootLogger } from "../lib/logger";
-import {
- highlightsEnvReady,
- runIndexedSearchHighlightsShadow,
- searchHighlightURLs,
-} from "./highlights";
-
-const CANONICAL_LOG = "search/highlights-shadow";
-const shadowLogger = createLogger({ silent: true });
-
-function sampled(requestId: string, rate: number): boolean {
- if (rate <= 0) return false;
- if (rate >= 1) return true;
-
- let hash = 2166136261;
- for (let i = 0; i < requestId.length; i++) {
- hash ^= requestId.charCodeAt(i);
- hash = Math.imul(hash, 16777619);
- }
- return (hash >>> 0) / 0x1_0000_0000 < rate;
-}
-
-export function createSearchHighlightsShadowRunner(
- canonicalLogger: Pick<Logger, "info" | "warn"> = rootLogger,
-): (options: {
- response: SearchV2Response;
- query: string;
- requestId: string;
- teamId: string;
- zeroDataRetention: boolean;
-}) => "skipped" | "started" {
- let inFlight = 0;
-
- return options => {
- if (
- options.zeroDataRetention ||
- !highlightsEnvReady() ||
- !sampled(options.requestId, config.HIGHLIGHT_SHADOW_RATE)
- ) {
- return "skipped";
- }
-
- const urls = searchHighlightURLs(options.response);
- const { query, requestId, teamId } = options;
- inFlight++;
- const startedAt = Date.now();
- void runIndexedSearchHighlightsShadow(urls, query, shadowLogger, requestId)
- .then(result => {
- canonicalLogger.info("Search highlights shadow completed", {
- canonicalLog: CANONICAL_LOG,
- outcome: result.succeeded ? "completed" : "failed",
- requestId,
- teamId,
- attempted: result.attempted,
- indexHits: result.indexHits,
- wouldReplace: result.replaced,
- failureReason: result.failureReason,
- timeTakenMs: Date.now() - startedAt,
- inFlight,
- sampleRate: config.HIGHLIGHT_SHADOW_RATE,
- });
- })
- .catch(error => {
- canonicalLogger.warn("Search highlights shadow failed", {
- canonicalLog: CANONICAL_LOG,
- outcome: "failed",
- requestId,
- teamId,
- errorType: error instanceof Error ? error.name : "unknown",
- timeTakenMs: Date.now() - startedAt,
- inFlight,
- sampleRate: config.HIGHLIGHT_SHADOW_RATE,
- });
- })
- .finally(() => {
- inFlight--;
- });
-
- return "started";
- };
-}
-
-export const runSearchHighlightsShadow = createSearchHighlightsShadowRunner();
diff --git a/apps/api/src/search/highlights.test.ts b/apps/api/src/search/highlights.test.ts
index 2f6c11976..3ee497d3f 100644
--- a/apps/api/src/search/highlights.test.ts
+++ b/apps/api/src/search/highlights.test.ts
@@ -3,6 +3,7 @@ vi.mock("../config", () => ({
GCS_INDEX_BUCKET_NAME: "index-bucket",
HIGHLIGHT_MODEL_URL: "https://highlight.test",
HIGHLIGHT_MODEL_TOKEN: "secret-token",
+ HIGHLIGHT_ROLLOUT_PERCENT: 0,
},
}));
@@ -10,9 +11,6 @@ vi.mock("../services", () => ({
useIndex: true,
normalizeURLForIndex: vi.fn((url: string) => url),
hashURL: vi.fn((url: string) => url),
- getIndexFromGCS: vi.fn(async (key: string) => ({
- html: `<main>${key}</main>`,
- })),
}));
vi.mock("../db/rpc", () => ({
@@ -25,31 +23,24 @@ vi.mock("../db/rpc", () => ({
]),
}));
-vi.mock("../lib/html-to-markdown", () => ({
- parseMarkdown: vi.fn(async (html: string) => `markdown:${html}`),
-}));
-
-vi.mock("../scraper/scrapeURL/lib/removeUnwantedElements", () => ({
- htmlTransform: vi.fn(async (html: string) => html),
+vi.mock("../lib/logger", () => ({
+ logger: {
+ info: vi.fn(),
+ warn: vi.fn(),
+ },
}));
vi.mock("./highlight-model", () => ({
- generateHighlightsBatch: vi.fn(),
generateHighlightsIndexedBatch: vi.fn(),
}));
-import {
- generateHighlightsBatch,
- generateHighlightsIndexedBatch,
-} from "./highlight-model";
+import { generateHighlightsIndexedBatch } from "./highlight-model";
import { config } from "../config";
-import { indexGetRecent5 } from "../db/rpc";
+import { logger as rootLogger } from "../lib/logger";
import {
- applyIndexedSearchHighlights,
- applySearchHighlights,
highlightsEnvReady,
- runIndexedSearchHighlightsShadow,
- searchHighlightURLs,
+ runIndexedSearchHighlights,
+ searchHighlightsMode,
} from "./highlights";
const logger = {
@@ -66,134 +57,7 @@ afterEach(() => {
vi.clearAllMocks();
});
-describe("runIndexedSearchHighlightsShadow", () => {
- it("resolves lightweight index references without loading page content", async () => {
- vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(
- new Map([["0", { highlights: [], markdown: "shadow highlight" }]]),
- );
- const response = {
- web: [{ url: "https://first.test", description: "fallback" }],
- news: [{ url: "https://second.test", snippet: "fallback" }],
- } as any;
- const urls = searchHighlightURLs(response);
-
- const result = await runIndexedSearchHighlightsShadow(
- urls,
- "query",
- logger,
- "request-1",
- );
-
- expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith(
- "query",
- [
- {
- id: "0",
- url: "https://first.test",
- indexObject: "index:https://first.test.json",
- },
- {
- id: "1",
- url: "https://second.test",
- indexObject: "index:https://second.test.json",
- },
- ],
- {
- logger,
- logPayload: false,
- requestId: "request-1",
- timeoutMs: null,
- onFailure: expect.any(Function),
- },
- );
- expect(result).toEqual({
- attempted: 2,
- indexHits: 2,
- replaced: 1,
- succeeded: true,
- });
- });
-});
-
-describe("applyIndexedSearchHighlights", () => {
- it("uses the shadow indexed request path and applies web and news responses by ID", async () => {
- vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(
- new Map([
- ["0", { highlights: [], markdown: "first highlight" }],
- ["1", { highlights: [], markdown: "second highlight" }],
- ]),
- );
- const response = {
- web: [{ url: "https://first.test", description: "first fallback" }],
- news: [{ url: "https://second.test", snippet: "second fallback" }],
- } as any;
-
- const result = await applyIndexedSearchHighlights(
- response,
- "query",
- logger,
- "request-1",
- );
-
- expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith(
- "query",
- [
- {
- id: "0",
- url: "https://first.test",
- indexObject: "index:https://first.test.json",
- },
- {
- id: "1",
- url: "https://second.test",
- indexObject: "index:https://second.test.json",
- },
- ],
- {
- logger,
- logPayload: false,
- requestId: "request-1",
- timeoutMs: null,
- onFailure: expect.any(Function),
- },
- );
- expect(generateHighlightsBatch).not.toHaveBeenCalled();
- expect(response.web[0].description).toBe("first highlight");
- expect(response.news[0].snippet).toBe("second highlight");
- expect(result).toEqual({
- attempted: 2,
- indexHits: 2,
- replaced: 2,
- succeeded: true,
- });
- });
-
- it("preserves provider snippets when the indexed Chain request fails", async () => {
- vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(null);
- const response = {
- web: [{ url: "https://first.test", description: "first fallback" }],
- news: [{ url: "https://second.test", snippet: "second fallback" }],
- } as any;
-
- const result = await applyIndexedSearchHighlights(
- response,
- "query",
- logger,
- "request-1",
- );
-
- expect(response.web[0].description).toBe("first fallback");
- expect(response.news[0].snippet).toBe("second fallback");
- expect(result).toEqual({
- attempted: 2,
- indexHits: 2,
- replaced: 0,
- succeeded: false,
- });
- });
-});
-
-describe("applySearchHighlights", () => {
+describe("runIndexedSearchHighlights", () => {
it("enables the in-cluster service without requiring a bearer token", () => {
const token = config.HIGHLIGHT_MODEL_TOKEN;
config.HIGHLIGHT_MODEL_TOKEN = undefined;
@@ -205,8 +69,8 @@ describe("applySearchHighlights", () => {
}
});
- it("sends indexed web and news results in one batch and applies responses by ID", async () => {
- vi.mocked(generateHighlightsBatch).mockResolvedValue(
+ it("uses indexed references and applies web and news responses", async () => {
+ vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(
new Map([
["0", { highlights: [], markdown: "first highlight" }],
["1", { highlights: [], markdown: "second highlight" }],
@@ -217,23 +81,31 @@ describe("applySearchHighlights", () => {
news: [{ url: "https://second.test", snippet: "second fallback" }],
} as any;
- const result = await applySearchHighlights(response, "query", logger);
+ const result = await runIndexedSearchHighlights(response, "query", logger, {
+ mode: "apply",
+ requestId: "request-1",
+ teamId: "team-1",
+ });
- expect(generateHighlightsBatch).toHaveBeenCalledTimes(1);
- expect(generateHighlightsBatch).toHaveBeenCalledWith(
+ expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith(
"query",
[
{
id: "0",
- markdown: "markdown:<main>index:https://first.test.json</main>",
+ url: "https://first.test",
+ indexObject: "index:https://first.test.json",
},
{
id: "1",
- markdown: "markdown:<main>index:https://second.test.json</main>",
+ url: "https://second.test",
+ indexObject: "index:https://second.test.json",
},
],
{
logger,
+ logPayload: false,
+ requestId: "request-1",
+ timeoutMs: 3000,
onFailure: expect.any(Function),
},
);
@@ -245,69 +117,24 @@ describe("applySearchHighlights", () => {
replaced: 2,
succeeded: true,
});
- });
-
- it("preserves individual fallbacks for missing and empty batch pages", async () => {
- vi.mocked(generateHighlightsBatch).mockResolvedValue(
- new Map([["0", { highlights: [], markdown: " " }]]),
+ expect(logger.info).toHaveBeenCalledWith(
+ "Search highlights completed",
+ expect.objectContaining({ mode: "apply", applied: 2 }),
);
- const response = {
- web: [
- { url: "https://first.test", description: "first fallback" },
- { url: "https://second.test", description: "second fallback" },
- ],
- } as any;
-
- const result = await applySearchHighlights(response, "query", logger);
-
- expect(response.web.map((item: any) => item.description)).toEqual([
- "first fallback",
- "second fallback",
- ]);
- expect(result).toEqual({
- attempted: 2,
- indexHits: 2,
- replaced: 0,
- succeeded: true,
- });
});
- it("preserves every fallback when the batch request fails", async () => {
- vi.mocked(generateHighlightsBatch).mockResolvedValue(null);
- const response = {
- web: [
- { url: "https://first.test", description: "first fallback" },
- { url: "https://second.test", description: "second fallback" },
- ],
- } as any;
-
- const result = await applySearchHighlights(response, "query", logger);
-
- expect(response.web.map((item: any) => item.description)).toEqual([
- "first fallback",
- "second fallback",
- ]);
- expect(result).toEqual({
- attempted: 2,
- indexHits: 2,
- replaced: 0,
- succeeded: false,
- });
- });
-
- it("runs without mutating the response or logging payloads in shadow mode", async () => {
- vi.mocked(generateHighlightsBatch).mockResolvedValue(
+ it("runs the same indexed path without mutating the response in shadow mode", async () => {
+ vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(
new Map([["0", { highlights: [], markdown: "shadow highlight" }]]),
);
const response = {
web: [{ url: "https://first.test", description: "fallback" }],
} as any;
- const result = await applySearchHighlights(response, "query", logger, {
- applyResults: false,
- suppressSummaryLog: true,
- suppressPayloadLog: true,
- allowLegacyFallback: false,
+ const result = await runIndexedSearchHighlights(response, "query", logger, {
+ mode: "shadow",
+ requestId: "request-1",
+ teamId: "team-1",
});
expect(response.web[0].description).toBe("fallback");
@@ -317,42 +144,71 @@ describe("applySearchHighlights", () => {
replaced: 1,
succeeded: true,
});
- expect(generateHighlightsBatch).toHaveBeenCalledWith(
+ expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith(
"query",
expect.any(Array),
- {
- logger,
+ expect.objectContaining({
+ logger: expect.objectContaining({ silent: true }),
logPayload: false,
- allowLegacyFallback: false,
- onFailure: expect.any(Function),
- },
+ }),
);
- expect(logger.info).not.toHaveBeenCalledWith(
- "Search highlights applied",
- expect.anything(),
+ expect(rootLogger.info).toHaveBeenCalledWith(
+ "Search highlights completed",
+ expect.objectContaining({ mode: "shadow", wouldApply: 1 }),
);
+ expect(logger.info).not.toHaveBeenCalled();
});
- it("omits result URLs from shadow lookup failures", async () => {
- vi.mocked(indexGetRecent5).mockRejectedValueOnce(
- new Error("lookup failed"),
- );
+ it("preserves provider snippets when the indexed request fails", async () => {
+ vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(null);
const response = {
- web: [{ url: "https://private.test", description: "fallback" }],
+ web: [{ url: "https://first.test", description: "first fallback" }],
} as any;
- await applySearchHighlights(response, "query", logger, {
- applyResults: false,
- suppressSummaryLog: true,
- suppressPayloadLog: true,
- allowLegacyFallback: false,
+ const result = await runIndexedSearchHighlights(response, "query", logger, {
+ mode: "apply",
+ requestId: "request-1",
+ teamId: "team-1",
});
- expect(logger.warn).toHaveBeenCalledWith(
- "highlights: index lookup failed",
- {
- error: "lookup failed",
- },
+ expect(response.web[0].description).toBe("first fallback");
+ expect(result).toEqual({
+ attempted: 1,
+ indexHits: 1,
+ replaced: 0,
+ succeeded: false,
+ });
+ });
+});
+
+describe("searchHighlightsMode", () => {
+ const base = {
+ cohortKey: "api-key:123",
+ rolloutPercent: 0,
+ };
+
+ it("applies omitted highlights for MCP and CLI callers", () => {
+ expect(searchHighlightsMode({ ...base, origin: "mcp" })).toBe("apply");
+ expect(searchHighlightsMode({ ...base, origin: "mcp-fastmcp" })).toBe(
+ "apply",
+ );
+ expect(searchHighlightsMode({ ...base, integration: "cli" })).toBe("apply");
+ });
+
+ it("honors explicit response selection before caller defaults", () => {
+ expect(searchHighlightsMode({ ...base, requested: true })).toBe("apply");
+ expect(
+ searchHighlightsMode({ ...base, requested: false, origin: "mcp" }),
+ ).toBe("shadow");
+ });
+
+ it("uses the rollout percentage for other callers", () => {
+ expect(searchHighlightsMode({ ...base, rolloutPercent: 0 })).toBe("shadow");
+ expect(searchHighlightsMode({ ...base, rolloutPercent: 100 })).toBe(
+ "apply",
);
+
+ const first = searchHighlightsMode({ ...base, rolloutPercent: 50 });
+ expect(searchHighlightsMode({ ...base, rolloutPercent: 50 })).toBe(first);
});
});
diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts
index a8448c7c1..1dd158c84 100644
--- a/apps/api/src/search/highlights.ts
+++ b/apps/api/src/search/highlights.ts
@@ -1,28 +1,24 @@
-import type { Logger } from "winston";
+import { createLogger, type Logger } from "winston";
import { SearchV2Response } from "../lib/entities";
-import {
- normalizeURLForIndex,
- hashURL,
- getIndexFromGCS,
- useIndex,
-} from "../services";
+import { normalizeURLForIndex, hashURL, useIndex } from "../services";
import { indexGetRecent5 } from "../db/rpc";
-import { parseMarkdown } from "../lib/html-to-markdown";
-import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";
-import type { ScrapeOptions } from "../controllers/v2/types";
import {
- generateHighlightsBatch,
generateHighlightsIndexedBatch,
type HighlightIndexedPage,
} from "./highlight-model";
import type { HighlightFailureReason } from "./highlight-model";
import { config } from "../config";
+import { logger as rootLogger } from "../lib/logger";
// How far back into the index we're willing to reach for highlight source text.
const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
+const HIGHLIGHTS_API_TIMEOUT_MS = 3000;
+const shadowLogger = createLogger({ silent: true });
+
+type SearchHighlightsMode = "apply" | "shadow";
/**
- * Whether the deployment has every dependency the highlights beta needs: the
+ * Whether the deployment has every dependency indexed highlights need: the
* index DB (to find cached content), the GCS index bucket (to fetch it), and the
* highlight model service URL (to score it). Missing any => silently skip.
*/
@@ -32,66 +28,43 @@ export function highlightsEnvReady(): boolean {
);
}
-// Mirrors scrapeURLWithIndex: prefer the newest 2xx entry unless it sits behind
-// this many more-recent error entries, in which case we surface the newest one.
-const ERROR_COUNT_TO_REGISTER = 3;
-
-// This whole module runs out-of-line from scrapeURL on purpose: it reads
-// already-indexed content directly from the index DB + GCS instead of routing
-// through the scrape engine. That keeps highlight generation off the critical
-// scrape path and lets us experiment with latency freely.
+function sampled(cohortKey: string, percent: number): boolean {
+ if (percent <= 0) return false;
+ if (percent >= 100) return true;
-/**
- * Fetch the most recent indexed markdown for a URL within the last 30 days.
- * Returns null when the URL isn't in the index (or the lookup fails) so callers
- * can fall back to the provider snippet.
- */
-async function getIndexedMarkdownForURL(
- url: string,
- logger: Logger,
- logUrl = true,
-): Promise<string | null> {
- const indexRef = await getIndexObjectForURL(url, logger, logUrl);
- if (!indexRef) {
- return null;
+ let hash = 2166136261;
+ for (let i = 0; i < cohortKey.length; i++) {
+ hash ^= cohortKey.charCodeAt(i);
+ hash = Math.imul(hash, 16777619);
}
+ return ((hash >>> 0) / 0x1_0000_0000) * 100 < percent;
+}
- try {
- const doc = await getIndexFromGCS(
- indexRef.name,
- logger.child({ module: "search/highlights", method: "getIndexFromGCS" }),
- { indexCreatedAt: indexRef.createdAt },
- );
- if (!doc || !doc.html) {
- return null;
- }
-
- // Skip raw base64 PDFs — they aren't useful as highlight source text.
- if (typeof doc.html === "string" && doc.html.startsWith("JVBERi")) {
- return null;
- }
-
- // The index stores rawHtml, so we must run the same cleaning the scrape
- // pipeline does (strip <style>/<script>/nav, extract main content) before
- // converting to markdown — otherwise CSS/JS leaks in and pollutes the
- // highlight source text.
- const cleanedHtml = await htmlTransform(doc.html, url, {
- onlyMainContent: true,
- includeTags: [],
- excludeTags: [],
- } as unknown as ScrapeOptions);
-
- const markdown = await parseMarkdown(cleanedHtml, { logger });
- return markdown && markdown.trim() !== "" ? markdown : null;
- } catch (error) {
- logger.warn("highlights: index content load failed", {
- error: error instanceof Error ? error.message : String(error),
- ...(logUrl ? { url } : {}),
- });
- return null;
+export function searchHighlightsMode(options: {
+ requested?: boolean;
+ origin?: string | null;
+ integration?: string | null;
+ cohortKey: string;
+ rolloutPercent: number;
+}): SearchHighlightsMode {
+ if (options.requested === true) return "apply";
+ if (options.requested === false) return "shadow";
+
+ const origin = (options.origin ?? "").toLowerCase();
+ const integration = (options.integration ?? "").toLowerCase();
+ if (integration === "cli" || origin === "cli" || origin.startsWith("mcp")) {
+ return "apply";
}
+
+ return sampled(options.cohortKey, options.rolloutPercent)
+ ? "apply"
+ : "shadow";
}
+// Mirrors scrapeURLWithIndex: prefer the newest 2xx entry unless it sits behind
+// this many more-recent error entries, in which case we surface the newest one.
+const ERROR_COUNT_TO_REGISTER = 3;
+
async function getIndexObjectForURL(
url: string,
logger: Logger,
@@ -176,15 +149,12 @@ function indexedSearchHighlightTargets(
return targets;
}
-export function searchHighlightURLs(response: SearchV2Response): string[] {
- return indexedSearchHighlightTargets(response).map(target => target.url);
-}
-
-async function runIndexedSearchHighlights(
+async function generateIndexedSearchHighlights(
targets: IndexedSearchHighlightTarget[],
query: string,
logger: Logger,
requestId: string,
+ applyResults: boolean,
): Promise<{
attempted: number;
indexHits: number;
@@ -211,7 +181,7 @@ async function runIndexedSearchHighlights(
logger,
logPayload: false,
requestId,
- timeoutMs: null,
+ timeoutMs: HIGHLIGHTS_API_TIMEOUT_MS,
onFailure: reason => {
failureReason = reason;
},
@@ -221,7 +191,9 @@ async function runIndexedSearchHighlights(
for (const page of pages) {
const highlight = results.get(page.id)?.markdown;
if (!highlight?.trim()) continue;
- targets[Number(page.id)].apply?.(highlight);
+ if (applyResults) {
+ targets[Number(page.id)].apply?.(highlight);
+ }
replaced++;
}
}
@@ -235,174 +207,38 @@ async function runIndexedSearchHighlights(
};
}
-export async function applyIndexedSearchHighlights(
+export async function runIndexedSearchHighlights(
response: SearchV2Response,
query: string,
logger: Logger,
- requestId: string,
-): ReturnType<typeof runIndexedSearchHighlights> {
+ options: {
+ mode: SearchHighlightsMode;
+ requestId: string;
+ teamId: string;
+ },
+): ReturnType<typeof generateIndexedSearchHighlights> {
const start = Date.now();
- const result = await runIndexedSearchHighlights(
+ const result = await generateIndexedSearchHighlights(
indexedSearchHighlightTargets(response),
query,
- logger,
- requestId,
+ options.mode === "shadow" ? shadowLogger : logger,
+ options.requestId,
+ options.mode === "apply",
);
- logger.info("Search highlights applied", {
+ const summaryLogger = options.mode === "shadow" ? rootLogger : logger;
+ summaryLogger.info("Search highlights completed", {
+ canonicalLog: "search/highlights",
+ mode: options.mode,
+ outcome: result.succeeded ? "completed" : "failed",
+ requestId: options.requestId,
+ teamId: options.teamId,
attempted: result.attempted,
indexHits: result.indexHits,
- replaced: result.replaced,
+ ...(options.mode === "apply"
+ ? { applied: result.replaced }
+ : { wouldApply: result.replaced }),
+ failureReason: result.failureReason,
timeTakenMs: Date.now() - start,
});
return result;
}
-
-export function runIndexedSearchHighlightsShadow(
- urls: string[],
- query: string,
- logger: Logger,
- requestId: string,
-): ReturnType<typeof runIndexedSearchHighlights> {
- return runIndexedSearchHighlights(
- urls.map(url => ({ url })),
- query,
- logger,
- requestId,
- );
-}
-
-/**
- * For each search result: look up the URL in our index (last 30 days), and if
- * present, replace the provider snippet with query-relevant highlights generated
- * from the indexed content. Index lookups run in parallel; each hit's full
- * markdown pages are sent to the highlight model service in one batch, which
- * returns each page's selected highlights reassembled into a single markdown
- * document. Mutates `response` in place. Results not in the index keep their
- * original snippet.
- */
-export async function applySearchHighlights(
- response: SearchV2Response,
- query: string,
- logger: Logger,
- options: {
- applyResults?: boolean;
- suppressSummaryLog?: boolean;
- suppressPayloadLog?: boolean;
- allowLegacyFallback?: boolean;
- requestId?: string;
- } = {},
-): Promise<{
- attempted: number;
- indexHits: number;
- replaced: number;
- succeeded: boolean;
- failureReason?: HighlightFailureReason;
-}> {
- const start = Date.now();
- const applyResults = options.applyResults ?? true;
-
- // Collect every result we could highlight, each with a setter for its snippet
- // field: web results carry it in `description`, news results in `snippet`.
- const targets: { url: string; apply: (h: string) => void }[] = [];
- for (const result of response.web ?? []) {
- if (!result.url) continue;
- targets.push({
- url: result.url,
- apply: h => {
- result.description = h;
- },
- });
- }
- for (const result of response.news ?? []) {
- if (!result.url) continue;
- targets.push({
- url: result.url,
- apply: h => {
- result.snippet = h;
- },
- });
- }
-
- const attempted = targets.length;
- if (attempted === 0) {
- return { attempted, indexHits: 0, replaced: 0, succeeded: true };
- }
-
- // Look up indexed markdown for every URL in parallel, keeping the markdown for
- // each hit so we can send it to the highlight model service.
- const markdowns = await Promise.all(
- targets.map(t =>
- getIndexedMarkdownForURL(t.url, logger, !options.suppressPayloadLog),
- ),
- );
- const hits: {
- apply: (h: string) => void;
- markdown: string;
- }[] = [];
- markdowns.forEach((markdown, i) => {
- if (!markdown) return;
- hits.push({ apply: targets[i].apply, markdown });
- });
- const indexHits = hits.length;
-
- // Send every hit in one request. IDs are local batch indexes, so a missing or
- // empty response only falls back the corresponding provider snippet.
- let replaced = 0;
- let succeeded = true;
- let failureReason: HighlightFailureReason | undefined;
- if (indexHits > 0) {
- const results = await generateHighlightsBatch(
- query,
- hits.map((hit, index) => ({
- id: String(index),
- markdown: hit.markdown,
- })),
- options.suppressPayloadLog || options.allowLegacyFallback === false
- ? {
- logger,
- logPayload: !options.suppressPayloadLog,
- allowLegacyFallback: options.allowLegacyFallback,
- ...(options.requestId ? { requestId: options.requestId } : {}),
- onFailure: reason => {
- failureReason = reason;
- },
- }
- : {
- logger,
- ...(options.requestId ? { requestId: options.requestId } : {}),
- onFailure: reason => {
- failureReason = reason;
- },
- },
- );
- succeeded = results !== null;
- if (results) {
- hits.forEach((hit, index) => {
- const snippet = results.get(String(index))?.markdown;
- if (snippet?.trim()) {
- if (applyResults) {
- hit.apply(snippet);
- }
- replaced++;
- }
- });
- }
- }
-
- if (!options.suppressSummaryLog) {
- logger.info("Search highlights applied", {
- attempted,
- indexHits,
- replaced,
- timeTakenMs: Date.now() - start,
- });
- }
-
- return {
- attempted,
- indexHits,
- replaced,
- succeeded,
- ...(failureReason ? { failureReason } : {}),
- };
-}