Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): move highlight shadow preprocessing off API (#4035)

Gergő Móriczbb9175fdf11f
diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.ts
index ffde591d4..bf2a5ccb1 100644
--- a/apps/api/src/search/highlight-model.test.ts
+++ b/apps/api/src/search/highlight-model.test.ts
@@ -5,7 +5,10 @@ vi.mock("../config", () => ({
},
}));
-import { generateHighlightsBatch } from "./highlight-model";
+import {
+ generateHighlightsBatch,
+ generateHighlightsIndexedBatch,
+} from "./highlight-model";
import { config } from "../config";
const logger = {
@@ -33,6 +36,35 @@ afterEach(() => {
});
describe("generateHighlightsBatch", () => {
+ it("posts lightweight index references to the indexed Stage 1 endpoint", async () => {
+ const fetchMock = mockFetchOnce({ pages: [] });
+
+ await generateHighlightsIndexedBatch(
+ "q1",
+ [
+ {
+ id: "0",
+ url: "https://first.test/path",
+ indexObject: "index-object.json",
+ },
+ ],
+ { logger, logPayload: false, requestId: "request-1" },
+ );
+
+ const [url, init] = fetchMock.mock.calls[0];
+ expect(url).toBe("https://highlight.test/batch_highlight_indexed");
+ expect(JSON.parse(init.body)).toEqual({
+ query: "q1",
+ pages: [
+ {
+ id: "0",
+ url: "https://first.test/path",
+ indexObject: "index-object.json",
+ },
+ ],
+ });
+ });
+
it("posts every page to one /batch_highlight call with the bearer token", async () => {
const fetchMock = mockFetchOnce({ pages: [] });
diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.ts
index 412431ff8..e2a06110c 100644
--- a/apps/api/src/search/highlight-model.ts
+++ b/apps/api/src/search/highlight-model.ts
@@ -61,6 +61,12 @@ interface HighlightPage {
markdown: string;
}
+export interface HighlightIndexedPage {
+ id: string;
+ url: string;
+ indexObject: string;
+}
+
// Result for one page in the batch: the service's highlight entries plus the
// reassembled markdown document.
interface HighlightResult {
@@ -149,16 +155,20 @@ function requestHeaders(requestId?: string): Record<string, string> {
* its provider snippet without discarding successful pages. Returns null when
* the whole call fails.
*/
-export async function generateHighlightsBatch(
+interface HighlightBatchOptions {
+ logger: Logger;
+ logPayload?: boolean;
+ allowLegacyFallback?: boolean;
+ requestId?: string;
+ onFailure?: (reason: HighlightFailureReason) => void;
+}
+
+async function generateHighlightsBatchRequest(
+ endpoint: "/batch_highlight" | "/batch_highlight_indexed",
query: string,
- pages: HighlightPage[],
- opts: {
- logger: Logger;
- logPayload?: boolean;
- allowLegacyFallback?: boolean;
- requestId?: string;
- onFailure?: (reason: HighlightFailureReason) => void;
- },
+ pages: Array<HighlightPage | HighlightIndexedPage>,
+ opts: HighlightBatchOptions,
+ legacyPages?: HighlightPage[],
): Promise<Map<string, HighlightResult> | null> {
if (pages.length === 0) {
return new Map();
@@ -170,7 +180,7 @@ export async function generateHighlightsBatch(
try {
const baseUrl = config.HIGHLIGHT_MODEL_URL!.replace(/\/$/, "");
const res = await fetchBatchWithRetry(
- `${baseUrl}/batch_highlight`,
+ `${baseUrl}${endpoint}`,
{
method: "POST",
headers: requestHeaders(opts.requestId),
@@ -186,6 +196,7 @@ export async function generateHighlightsBatch(
// endpoint, whose /batch_highlight contract uses {requests: [...]}. Keep
// highlights available until infra switches the URL to GCP Stage 1.
if (
+ legacyPages &&
opts.allowLegacyFallback !== false &&
(res.status === 400 || res.status === 404)
) {
@@ -196,7 +207,7 @@ export async function generateHighlightsBatch(
return await generateLegacyHighlightsBatch(
baseUrl,
query,
- pages,
+ legacyPages,
controller.signal,
opts,
);
@@ -253,6 +264,33 @@ export async function generateHighlightsBatch(
}
}
+export async function generateHighlightsBatch(
+ query: string,
+ pages: HighlightPage[],
+ opts: HighlightBatchOptions,
+): Promise<Map<string, HighlightResult> | null> {
+ return generateHighlightsBatchRequest(
+ "/batch_highlight",
+ query,
+ pages,
+ opts,
+ pages,
+ );
+}
+
+export async function generateHighlightsIndexedBatch(
+ query: string,
+ pages: HighlightIndexedPage[],
+ opts: Omit<HighlightBatchOptions, "allowLegacyFallback">,
+): Promise<Map<string, HighlightResult> | null> {
+ return generateHighlightsBatchRequest(
+ "/batch_highlight_indexed",
+ query,
+ pages,
+ { ...opts, allowLegacyFallback: false },
+ );
+}
+
async function generateLegacyHighlightsBatch(
baseUrl: string,
query: string,
diff --git a/apps/api/src/search/highlights-shadow.test.ts b/apps/api/src/search/highlights-shadow.test.ts
index aa8208441..dfebfd18e 100644
--- a/apps/api/src/search/highlights-shadow.test.ts
+++ b/apps/api/src/search/highlights-shadow.test.ts
@@ -7,11 +7,16 @@ vi.mock("../config", () => ({
vi.mock("./highlights", () => ({
highlightsEnvReady: vi.fn(() => true),
- applySearchHighlights: vi.fn(),
+ searchHighlightURLs: vi.fn(() => []),
+ runIndexedSearchHighlightsShadow: vi.fn(),
}));
import { config } from "../config";
-import { applySearchHighlights, highlightsEnvReady } from "./highlights";
+import {
+ highlightsEnvReady,
+ runIndexedSearchHighlightsShadow,
+ searchHighlightURLs,
+} from "./highlights";
import { createSearchHighlightsShadowRunner } from "./highlights-shadow";
const logger = {
@@ -31,7 +36,7 @@ afterEach(() => {
describe("runSearchHighlightsShadow", () => {
it("runs without applying results and emits a content-free canonical log", async () => {
- vi.mocked(applySearchHighlights).mockResolvedValue({
+ vi.mocked(runIndexedSearchHighlightsShadow).mockResolvedValue({
attempted: 10,
indexHits: 7,
replaced: 6,
@@ -49,17 +54,12 @@ describe("runSearchHighlightsShadow", () => {
).toBe("started");
await new Promise(resolve => setImmediate(resolve));
- expect(applySearchHighlights).toHaveBeenCalledWith(
- {},
+ expect(searchHighlightURLs).toHaveBeenCalledWith({});
+ expect(runIndexedSearchHighlightsShadow).toHaveBeenCalledWith(
+ [],
"private query",
expect.objectContaining({ silent: true }),
- {
- applyResults: false,
- suppressSummaryLog: true,
- suppressPayloadLog: true,
- allowLegacyFallback: false,
- requestId: "request-1",
- },
+ "request-1",
);
expect(logger.info).toHaveBeenCalledWith(
"Search highlights shadow completed",
@@ -84,7 +84,7 @@ describe("runSearchHighlightsShadow", () => {
replaced: number;
succeeded: boolean;
}) => void;
- vi.mocked(applySearchHighlights).mockReturnValue(
+ vi.mocked(runIndexedSearchHighlightsShadow).mockReturnValue(
new Promise(resolve => {
finish = resolve;
}),
@@ -116,7 +116,7 @@ describe("runSearchHighlightsShadow", () => {
});
it("emits the content-free failure category", async () => {
- vi.mocked(applySearchHighlights).mockResolvedValue({
+ vi.mocked(runIndexedSearchHighlightsShadow).mockResolvedValue({
attempted: 3,
indexHits: 1,
replaced: 0,
@@ -165,6 +165,6 @@ describe("runSearchHighlightsShadow", () => {
runSearchHighlightsShadow({ ...input, zeroDataRetention: false }),
).toBe("skipped");
- expect(applySearchHighlights).not.toHaveBeenCalled();
+ expect(runIndexedSearchHighlightsShadow).not.toHaveBeenCalled();
});
});
diff --git a/apps/api/src/search/highlights-shadow.ts b/apps/api/src/search/highlights-shadow.ts
index 5ed9c9633..3c9f6386a 100644
--- a/apps/api/src/search/highlights-shadow.ts
+++ b/apps/api/src/search/highlights-shadow.ts
@@ -2,7 +2,11 @@ import { createLogger, type Logger } from "winston";
import type { SearchV2Response } from "../lib/entities";
import { config } from "../config";
import { logger as rootLogger } from "../lib/logger";
-import { applySearchHighlights, highlightsEnvReady } from "./highlights";
+import {
+ highlightsEnvReady,
+ runIndexedSearchHighlightsShadow,
+ searchHighlightURLs,
+} from "./highlights";
const CANONICAL_LOG = "search/highlights-shadow";
const shadowLogger = createLogger({ silent: true });
@@ -53,21 +57,17 @@ export function createSearchHighlightsShadowRunner(
return "dropped";
}
+ const urls = searchHighlightURLs(options.response);
+ const { query, requestId, teamId } = options;
inFlight++;
const startedAt = Date.now();
- void applySearchHighlights(options.response, options.query, shadowLogger, {
- applyResults: false,
- suppressSummaryLog: true,
- suppressPayloadLog: true,
- allowLegacyFallback: false,
- requestId: options.requestId,
- })
+ void runIndexedSearchHighlightsShadow(urls, query, shadowLogger, requestId)
.then(result => {
canonicalLogger.info("Search highlights shadow completed", {
canonicalLog: CANONICAL_LOG,
outcome: result.succeeded ? "completed" : "failed",
- requestId: options.requestId,
- teamId: options.teamId,
+ requestId,
+ teamId,
attempted: result.attempted,
indexHits: result.indexHits,
wouldReplace: result.replaced,
@@ -82,8 +82,8 @@ export function createSearchHighlightsShadowRunner(
canonicalLogger.warn("Search highlights shadow failed", {
canonicalLog: CANONICAL_LOG,
outcome: "failed",
- requestId: options.requestId,
- teamId: options.teamId,
+ requestId,
+ teamId,
errorType: error instanceof Error ? error.name : "unknown",
timeTakenMs: Date.now() - startedAt,
inFlight,
diff --git a/apps/api/src/search/highlights.test.ts b/apps/api/src/search/highlights.test.ts
index 5a677dbaf..0463b78b9 100644
--- a/apps/api/src/search/highlights.test.ts
+++ b/apps/api/src/search/highlights.test.ts
@@ -35,12 +35,21 @@ vi.mock("../scraper/scrapeURL/lib/removeUnwantedElements", () => ({
vi.mock("./highlight-model", () => ({
generateHighlightsBatch: vi.fn(),
+ generateHighlightsIndexedBatch: vi.fn(),
}));
-import { generateHighlightsBatch } from "./highlight-model";
+import {
+ generateHighlightsBatch,
+ generateHighlightsIndexedBatch,
+} from "./highlight-model";
import { config } from "../config";
import { indexGetRecent5 } from "../db/rpc";
-import { applySearchHighlights, highlightsEnvReady } from "./highlights";
+import {
+ applySearchHighlights,
+ highlightsEnvReady,
+ runIndexedSearchHighlightsShadow,
+ searchHighlightURLs,
+} from "./highlights";
const logger = {
info: vi.fn(),
@@ -56,6 +65,54 @@ afterEach(() => {
vi.clearAllMocks();
});
+describe("runIndexedSearchHighlightsShadow", () => {
+ it("resolves lightweight index references without loading page content", async () => {
+ vi.mocked(generateHighlightsIndexedBatch).mockResolvedValue(
+ new Map([["0", { highlights: [], markdown: "shadow highlight" }]]),
+ );
+ const response = {
+ web: [{ url: "https://first.test", description: "fallback" }],
+ news: [{ url: "https://second.test", snippet: "fallback" }],
+ } as any;
+ const urls = searchHighlightURLs(response);
+
+ const result = await runIndexedSearchHighlightsShadow(
+ urls,
+ "query",
+ logger,
+ "request-1",
+ );
+
+ expect(generateHighlightsIndexedBatch).toHaveBeenCalledWith(
+ "query",
+ [
+ {
+ id: "0",
+ url: "https://first.test",
+ indexObject: "index:https://first.test.json",
+ },
+ {
+ id: "1",
+ url: "https://second.test",
+ indexObject: "index:https://second.test.json",
+ },
+ ],
+ {
+ logger,
+ logPayload: false,
+ requestId: "request-1",
+ onFailure: expect.any(Function),
+ },
+ );
+ expect(result).toEqual({
+ attempted: 2,
+ indexHits: 2,
+ replaced: 1,
+ succeeded: true,
+ });
+ });
+});
+
describe("applySearchHighlights", () => {
it("enables the in-cluster service without requiring a bearer token", () => {
const token = config.HIGHLIGHT_MODEL_TOKEN;
diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts
index 53561cb97..a769b415b 100644
--- a/apps/api/src/search/highlights.ts
+++ b/apps/api/src/search/highlights.ts
@@ -10,7 +10,11 @@ import { indexGetRecent5 } from "../db/rpc";
import { parseMarkdown } from "../lib/html-to-markdown";
import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";
import type { ScrapeOptions } from "../controllers/v2/types";
-import { generateHighlightsBatch } from "./highlight-model";
+import {
+ generateHighlightsBatch,
+ generateHighlightsIndexedBatch,
+ type HighlightIndexedPage,
+} from "./highlight-model";
import type { HighlightFailureReason } from "./highlight-model";
import { config } from "../config";
@@ -47,6 +51,52 @@ async function getIndexedMarkdownForURL(
logger: Logger,
logUrl = true,
): Promise<string | null> {
+ const indexRef = await getIndexObjectForURL(url, logger, logUrl);
+ if (!indexRef) {
+ return null;
+ }
+
+ try {
+ const doc = await getIndexFromGCS(
+ indexRef.name,
+ logger.child({ module: "search/highlights", method: "getIndexFromGCS" }),
+ { indexCreatedAt: indexRef.createdAt },
+ );
+ if (!doc || !doc.html) {
+ return null;
+ }
+
+ // Skip raw base64 PDFs — they aren't useful as highlight source text.
+ if (typeof doc.html === "string" && doc.html.startsWith("JVBERi")) {
+ return null;
+ }
+
+ // The index stores rawHtml, so we must run the same cleaning the scrape
+ // pipeline does (strip <style>/<script>/nav, extract main content) before
+ // converting to markdown — otherwise CSS/JS leaks in and pollutes the
+ // highlight source text.
+ const cleanedHtml = await htmlTransform(doc.html, url, {
+ onlyMainContent: true,
+ includeTags: [],
+ excludeTags: [],
+ } as unknown as ScrapeOptions);
+
+ const markdown = await parseMarkdown(cleanedHtml, { logger });
+ return markdown && markdown.trim() !== "" ? markdown : null;
+ } catch (error) {
+ logger.warn("highlights: index content load failed", {
+ error: error instanceof Error ? error.message : String(error),
+ ...(logUrl ? { url } : {}),
+ });
+ return null;
+ }
+}
+
+async function getIndexObjectForURL(
+ url: string,
+ logger: Logger,
+ logUrl = true,
+): Promise<{ name: string; createdAt: string | null } | null> {
if (!useIndex) {
return null;
}
@@ -83,32 +133,10 @@ async function getIndexedMarkdownForURL(
? rows[0]
: rows[newest200Index];
- const doc = await getIndexFromGCS(
- selected.id + ".json",
- logger.child({ module: "search/highlights", method: "getIndexFromGCS" }),
- { indexCreatedAt: selected.created_at },
- );
- if (!doc || !doc.html) {
- return null;
- }
-
- // Skip raw base64 PDFs — they aren't useful as highlight source text.
- if (typeof doc.html === "string" && doc.html.startsWith("JVBERi")) {
- return null;
- }
-
- // The index stores rawHtml, so we must run the same cleaning the scrape
- // pipeline does (strip <style>/<script>/nav, extract main content) before
- // converting to markdown — otherwise CSS/JS leaks in and pollutes the
- // highlight source text.
- const cleanedHtml = await htmlTransform(doc.html, url, {
- onlyMainContent: true,
- includeTags: [],
- excludeTags: [],
- } as unknown as ScrapeOptions);
-
- const markdown = await parseMarkdown(cleanedHtml, { logger });
- return markdown && markdown.trim() !== "" ? markdown : null;
+ return {
+ name: selected.id + ".json",
+ createdAt: selected.created_at,
+ };
} catch (error) {
logger.warn("highlights: index lookup failed", {
error: error instanceof Error ? error.message : String(error),
@@ -118,6 +146,64 @@ async function getIndexedMarkdownForURL(
}
}
+export function searchHighlightURLs(response: SearchV2Response): string[] {
+ return [
+ ...(response.web ?? []).flatMap(result => (result.url ? [result.url] : [])),
+ ...(response.news ?? []).flatMap(result =>
+ result.url ? [result.url] : [],
+ ),
+ ];
+}
+
+export async function runIndexedSearchHighlightsShadow(
+ urls: string[],
+ query: string,
+ logger: Logger,
+ requestId: string,
+): Promise<{
+ attempted: number;
+ indexHits: number;
+ replaced: number;
+ succeeded: boolean;
+ failureReason?: HighlightFailureReason;
+}> {
+ const attempted = urls.length;
+ const resolved = await Promise.all(
+ urls.map(url => getIndexObjectForURL(url, logger, false)),
+ );
+ const pages: HighlightIndexedPage[] = [];
+ resolved.forEach((indexRef, index) => {
+ if (!indexRef) return;
+ pages.push({
+ id: String(index),
+ url: urls[index],
+ indexObject: indexRef.name,
+ });
+ });
+
+ let failureReason: HighlightFailureReason | undefined;
+ const results = await generateHighlightsIndexedBatch(query, pages, {
+ logger,
+ logPayload: false,
+ requestId,
+ onFailure: reason => {
+ failureReason = reason;
+ },
+ });
+ const replaced = results
+ ? Array.from(results.values()).filter(result => result.markdown.trim())
+ .length
+ : 0;
+
+ return {
+ attempted,
+ indexHits: pages.length,
+ replaced,
+ succeeded: results !== null,
+ ...(failureReason ? { failureReason } : {}),
+ };
+}
+
/**
* For each search result: look up the URL in our index (last 30 days), and if
* present, replace the provider snippet with query-relevant highlights generated