Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(search): batch query highlights (#3993)

Gergő Móriczb113bec5cc78
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 35c082082..8a672944d 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -159,9 +159,8 @@ const configSchema = z.object({
CLICKHOUSE_ANALYTICS_URL: z.string().optional(),
CLICKHOUSE_ANALYTICS_DATABASE: z.string().optional(),
- // Search highlights (beta): query-highlights model service. URL is the base
- // (e.g. https://firecrawl--query-highlights.modal.run); TOKEN is the bearer
- // token sent on every request.
+ // Search highlights (beta): highlighter service base URL. TOKEN is optional
+ // bearer auth for legacy/external services; the in-cluster service omits it.
HIGHLIGHT_MODEL_URL: z.string().optional(),
HIGHLIGHT_MODEL_TOKEN: z.string().optional(),
diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.ts
index 23cc19ba4..b70925d8a 100644
--- a/apps/api/src/search/highlight-model.test.ts
+++ b/apps/api/src/search/highlight-model.test.ts
@@ -5,7 +5,8 @@ vi.mock("../config", () => ({
},
}));
-import { generateHighlights } from "./highlight-model";
+import { generateHighlightsBatch } from "./highlight-model";
+import { config } from "../config";
const logger = {
info: vi.fn(),
@@ -30,34 +31,81 @@ afterEach(() => {
vi.clearAllMocks();
});
-describe("generateHighlights", () => {
- it("posts the query and full markdown to /highlight with the bearer token", async () => {
- const fetchMock = mockFetchOnce({ highlights: [], markdown: "" });
+describe("generateHighlightsBatch", () => {
+ it("posts every page to one /batch_highlight call with the bearer token", async () => {
+ const fetchMock = mockFetchOnce({ pages: [] });
- await generateHighlights("q1", "# Page\n\nbody text", { logger });
+ await generateHighlightsBatch(
+ "q1",
+ [
+ { id: "0", markdown: "# First\n\nbody text" },
+ { id: "1", markdown: "# Second\n\nmore text" },
+ ],
+ { logger },
+ );
expect(fetchMock).toHaveBeenCalledTimes(1);
const [url, init] = fetchMock.mock.calls[0];
- expect(url).toBe("https://highlight.test/highlight");
+ expect(url).toBe("https://highlight.test/batch_highlight");
expect(init.method).toBe("POST");
expect(init.headers.Authorization).toBe("Bearer secret-token");
const sent = JSON.parse(init.body);
- expect(sent).toEqual({ query: "q1", markdown: "# Page\n\nbody text" });
+ expect(sent).toEqual({
+ query: "q1",
+ pages: [
+ { id: "0", markdown: "# First\n\nbody text" },
+ { id: "1", markdown: "# Second\n\nmore text" },
+ ],
+ });
});
- it("returns the highlights and reassembled markdown from the service", async () => {
+ it("omits bearer auth for the in-cluster service when no token is configured", async () => {
+ const token = config.HIGHLIGHT_MODEL_TOKEN;
+ config.HIGHLIGHT_MODEL_TOKEN = undefined;
+ const fetchMock = mockFetchOnce({ pages: [] });
+
+ try {
+ await generateHighlightsBatch("q1", [{ id: "0", markdown: "markdown" }], {
+ logger,
+ });
+ } finally {
+ config.HIGHLIGHT_MODEL_TOKEN = token;
+ }
+
+ const [, init] = fetchMock.mock.calls[0];
+ expect(init.headers).toEqual({ "Content-Type": "application/json" });
+ });
+
+ it("returns successful pages keyed by ID and leaves missing pages absent", async () => {
mockFetchOnce({
- highlights: [
- { block_index: 0, score: 0.9 },
- { block_index: 2, score: 0.5 },
+ pages: [
+ {
+ id: "1",
+ cache: "hit",
+ output: {
+ highlights: [
+ { block_index: 0, score: 0.9 },
+ { block_index: 2, score: 0.5 },
+ ],
+ markdown: "reassembled answer",
+ },
+ },
+ { id: "invalid-without-output" },
],
- markdown: "reassembled answer",
});
- const out = await generateHighlights("q", "some markdown", { logger });
+ const out = await generateHighlightsBatch(
+ "q",
+ [
+ { id: "0", markdown: "missing" },
+ { id: "1", markdown: "some markdown" },
+ ],
+ { logger },
+ );
- expect(out).toEqual({
+ expect(out?.get("0")).toBeUndefined();
+ expect(out?.get("1")).toEqual({
highlights: [
{ block_index: 0, score: 0.9 },
{ block_index: 2, score: 0.5 },
@@ -66,37 +114,117 @@ describe("generateHighlights", () => {
});
});
- it("debug-logs the highlights array", async () => {
+ it("debug-logs batch coverage and highlights", async () => {
mockFetchOnce({
- highlights: [{ block_index: 1, score: 0.8 }],
- markdown: "x",
+ pages: [
+ {
+ id: "page-1",
+ output: {
+ highlights: [{ block_index: 1, score: 0.8 }],
+ markdown: "x",
+ },
+ },
+ ],
});
- await generateHighlights("q", "md", { logger });
+ await generateHighlightsBatch("q", [{ id: "page-1", markdown: "md" }], {
+ logger,
+ });
expect(logger.debug).toHaveBeenCalledWith(
- "query highlights",
+ "query highlights batch",
expect.objectContaining({
canonicalLog: "search/highlights",
- highlights: [{ block_index: 1, score: 0.8 }],
+ requestedPages: 1,
+ returnedPages: 1,
+ pages: [
+ {
+ id: "page-1",
+ highlights: [{ block_index: 1, score: 0.8 }],
+ },
+ ],
}),
);
});
- it("defaults missing fields to empty highlights and markdown", async () => {
+ it("defaults missing pages to an empty result map", async () => {
mockFetchOnce({});
- const out = await generateHighlights("q", "md", { logger });
+ const out = await generateHighlightsBatch(
+ "q",
+ [{ id: "0", markdown: "md" }],
+ { logger },
+ );
- expect(out).toEqual({ highlights: [], markdown: "" });
+ expect(out).toEqual(new Map());
+ });
+
+ it("does not call the service for an empty batch", async () => {
+ const fetchMock = vi.fn();
+ vi.stubGlobal("fetch", fetchMock);
+
+ const out = await generateHighlightsBatch("q", [], { logger });
+
+ expect(fetchMock).not.toHaveBeenCalled();
+ expect(out).toEqual(new Map());
});
it("returns null when the service errors", async () => {
mockFetchOnce({ error: "boom" }, false, 500);
- const out = await generateHighlights("q", "md", { logger });
+ const out = await generateHighlightsBatch(
+ "q",
+ [{ id: "0", markdown: "md" }],
+ { logger },
+ );
expect(out).toBeNull();
expect(logger.warn).toHaveBeenCalled();
});
+
+ it("falls back to legacy per-page calls while the old service URL is configured", async () => {
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce({
+ ok: false,
+ status: 400,
+ text: async () => "legacy batch contract",
+ })
+ .mockResolvedValueOnce({
+ ok: true,
+ status: 200,
+ json: async () => ({ markdown: "first legacy highlight" }),
+ text: async () => "",
+ })
+ .mockResolvedValueOnce({
+ ok: false,
+ status: 500,
+ json: async () => ({}),
+ text: async () => "page failed",
+ });
+ vi.stubGlobal("fetch", fetchMock);
+
+ const out = await generateHighlightsBatch(
+ "q",
+ [
+ { id: "0", markdown: "first" },
+ { id: "1", markdown: "second" },
+ ],
+ { logger },
+ );
+
+ expect(fetchMock).toHaveBeenCalledTimes(3);
+ expect(fetchMock.mock.calls.map(call => call[0])).toEqual([
+ "https://highlight.test/batch_highlight",
+ "https://highlight.test/highlight",
+ "https://highlight.test/highlight",
+ ]);
+ expect(out).toEqual(
+ new Map([["0", { highlights: [], markdown: "first legacy highlight" }]]),
+ );
+ expect(logger.warn).toHaveBeenCalledWith(
+ "legacy query highlight failed",
+ expect.objectContaining({ pageId: "1" }),
+ );
+ });
});
diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.ts
index 427c9543a..1feb53c09 100644
--- a/apps/api/src/search/highlight-model.ts
+++ b/apps/api/src/search/highlight-model.ts
@@ -1,12 +1,13 @@
import type { Logger } from "winston";
import { config } from "../config";
-// Query Highlights model service: given a page's full markdown and a query, the
-// service returns the query-relevant highlights plus those highlights
-// reassembled into a single markdown document (in document order). We push each
-// page through one `/highlight` call. Endpoint + token are config-gated
-// (HIGHLIGHT_MODEL_URL / HIGHLIGHT_MODEL_TOKEN); callers must confirm they're
-// set via highlightsEnvReady() before invoking.
+// Query Highlights model service: given full markdown pages and a query, the
+// service returns query-relevant highlights plus each page's highlights
+// reassembled into a single markdown document (in document order). All indexed
+// search results go through one `/batch_highlight` call. The endpoint is
+// URL-config-gated; callers must confirm it is set via highlightsEnvReady()
+// before invoking. Bearer auth remains optional for legacy/external services;
+// the in-cluster GCP Stage 1 service relies on cluster network isolation.
const REQUEST_TIMEOUT_MS = 30000;
@@ -25,58 +26,111 @@ interface HighlightResponse {
markdown?: string;
}
-// Result of one highlight call: the service's highlight entries plus the
+interface HighlightBatchPageResponse {
+ id?: string;
+ output?: HighlightResponse;
+}
+
+interface HighlightBatchResponse {
+ pages?: HighlightBatchPageResponse[];
+}
+
+interface HighlightPage {
+ id: string;
+ markdown: string;
+}
+
+// Result for one page in the batch: the service's highlight entries plus the
// reassembled markdown document.
interface HighlightResult {
highlights: Highlight[];
markdown: string;
}
+function requestHeaders(): Record<string, string> {
+ const headers: Record<string, string> = {
+ "Content-Type": "application/json",
+ };
+ if (config.HIGHLIGHT_MODEL_TOKEN) {
+ headers.Authorization = `Bearer ${config.HIGHLIGHT_MODEL_TOKEN}`;
+ }
+ return headers;
+}
+
/**
- * Generate query highlights for one page by sending its full markdown and the
- * query to the highlight model service. Returns the service's highlight entries
- * plus the reassembled markdown, or null when the call fails (so callers can
- * keep the provider snippet).
+ * Generate query highlights for every indexed result in one request. Results
+ * are keyed by the caller-provided page ID so a missing page can fall back to
+ * its provider snippet without discarding successful pages. Returns null when
+ * the whole call fails.
*/
-export async function generateHighlights(
+export async function generateHighlightsBatch(
query: string,
- markdown: string,
+ pages: HighlightPage[],
opts: { logger: Logger },
-): Promise<HighlightResult | null> {
+): Promise<Map<string, HighlightResult> | null> {
+ if (pages.length === 0) {
+ return new Map();
+ }
+
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
const start = Date.now();
try {
- const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/highlight`, {
+ const baseUrl = config.HIGHLIGHT_MODEL_URL!.replace(/\/$/, "");
+ const res = await fetch(`${baseUrl}/batch_highlight`, {
method: "POST",
- headers: {
- "Content-Type": "application/json",
- Authorization: `Bearer ${config.HIGHLIGHT_MODEL_TOKEN}`,
- },
- body: JSON.stringify({ query, markdown }),
+ headers: requestHeaders(),
+ body: JSON.stringify({ query, pages }),
signal: controller.signal,
});
if (!res.ok) {
const body = await res.text().catch(() => "");
+ // During rollout the configured service may still be the legacy Modal
+ // endpoint, whose /batch_highlight contract uses {requests: [...]}. Keep
+ // highlights available until infra switches the URL to GCP Stage 1.
+ if (res.status === 400 || res.status === 404) {
+ opts.logger.info("query highlights using legacy per-page fallback", {
+ canonicalLog: "search/highlights",
+ status: res.status,
+ });
+ return await generateLegacyHighlightsBatch(
+ baseUrl,
+ query,
+ pages,
+ controller.signal,
+ opts,
+ );
+ }
throw new Error(
`highlight model HTTP ${res.status}: ${body.slice(0, 200)}`,
);
}
- const data = (await res.json()) as HighlightResponse;
- const highlights = data.highlights ?? [];
+ const data = (await res.json()) as HighlightBatchResponse;
+ const results = new Map<string, HighlightResult>();
+ for (const page of data.pages ?? []) {
+ if (typeof page.id !== "string" || !page.output) continue;
+ results.set(page.id, {
+ highlights: page.output.highlights ?? [],
+ markdown: page.output.markdown ?? "",
+ });
+ }
- // Canonical log of the highlights array, debug-level only.
- opts.logger.debug("query highlights", {
+ opts.logger.debug("query highlights batch", {
canonicalLog: "search/highlights",
- highlights,
+ requestedPages: pages.length,
+ returnedPages: results.size,
+ pages: Array.from(results, ([id, result]) => ({
+ id,
+ highlights: result.highlights,
+ })),
elapsedMs: Date.now() - start,
});
- return { highlights, markdown: data.markdown ?? "" };
+ return results;
} catch (error) {
- opts.logger.warn("query highlights failed", {
+ opts.logger.warn("query highlights batch failed", {
canonicalLog: "search/highlights",
error: error instanceof Error ? error.message : String(error),
});
@@ -85,3 +139,46 @@ export async function generateHighlights(
clearTimeout(timer);
}
}
+
+async function generateLegacyHighlightsBatch(
+ baseUrl: string,
+ query: string,
+ pages: HighlightPage[],
+ signal: AbortSignal,
+ opts: { logger: Logger },
+): Promise<Map<string, HighlightResult>> {
+ const entries = await Promise.all(
+ pages.map(async page => {
+ try {
+ const res = await fetch(`${baseUrl}/highlight`, {
+ method: "POST",
+ headers: requestHeaders(),
+ body: JSON.stringify({ query, markdown: page.markdown }),
+ signal,
+ });
+ if (!res.ok) {
+ const body = await res.text().catch(() => "");
+ throw new Error(
+ `highlight model HTTP ${res.status}: ${body.slice(0, 200)}`,
+ );
+ }
+ const data = (await res.json()) as HighlightResponse;
+ return [
+ page.id,
+ {
+ highlights: data.highlights ?? [],
+ markdown: data.markdown ?? "",
+ },
+ ] as const;
+ } catch (error) {
+ opts.logger.warn("legacy query highlight failed", {
+ canonicalLog: "search/highlights",
+ pageId: page.id,
+ error: error instanceof Error ? error.message : String(error),
+ });
+ return null;
+ }
+ }),
+ );
+ return new Map(entries.filter(entry => entry !== null));
+}
diff --git a/apps/api/src/search/highlights.test.ts b/apps/api/src/search/highlights.test.ts
new file mode 100644
index 000000000..ccaa8791f
--- /dev/null
+++ b/apps/api/src/search/highlights.test.ts
@@ -0,0 +1,141 @@
+vi.mock("../config", () => ({
+ config: {
+ GCS_INDEX_BUCKET_NAME: "index-bucket",
+ HIGHLIGHT_MODEL_URL: "https://highlight.test",
+ HIGHLIGHT_MODEL_TOKEN: "secret-token",
+ },
+}));
+
+vi.mock("../services", () => ({
+ useIndex: true,
+ normalizeURLForIndex: vi.fn((url: string) => url),
+ hashURL: vi.fn((url: string) => url),
+ getIndexFromGCS: vi.fn(async (key: string) => ({
+ html: `<main>${key}</main>`,
+ })),
+}));
+
+vi.mock("../db/rpc", () => ({
+ indexGetRecent5: vi.fn(async ({ url_hash }: { url_hash: string }) => [
+ {
+ id: `index:${url_hash}`,
+ status: 200,
+ created_at: new Date("2026-07-11T00:00:00Z"),
+ },
+ ]),
+}));
+
+vi.mock("../lib/html-to-markdown", () => ({
+ parseMarkdown: vi.fn(async (html: string) => `markdown:${html}`),
+}));
+
+vi.mock("../scraper/scrapeURL/lib/removeUnwantedElements", () => ({
+ htmlTransform: vi.fn(async (html: string) => html),
+}));
+
+vi.mock("./highlight-model", () => ({
+ generateHighlightsBatch: vi.fn(),
+}));
+
+import { generateHighlightsBatch } from "./highlight-model";
+import { config } from "../config";
+import { applySearchHighlights, highlightsEnvReady } from "./highlights";
+
+const logger = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+ child: vi.fn(function () {
+ return this;
+ }),
+} as any;
+
+afterEach(() => {
+ vi.clearAllMocks();
+});
+
+describe("applySearchHighlights", () => {
+ it("enables the in-cluster service without requiring a bearer token", () => {
+ const token = config.HIGHLIGHT_MODEL_TOKEN;
+ config.HIGHLIGHT_MODEL_TOKEN = undefined;
+
+ try {
+ expect(highlightsEnvReady()).toBe(true);
+ } finally {
+ config.HIGHLIGHT_MODEL_TOKEN = token;
+ }
+ });
+
+ it("sends indexed web and news results in one batch and applies responses by ID", async () => {
+ vi.mocked(generateHighlightsBatch).mockResolvedValue(
+ new Map([
+ ["0", { highlights: [], markdown: "first highlight" }],
+ ["1", { highlights: [], markdown: "second highlight" }],
+ ]),
+ );
+ const response = {
+ web: [{ url: "https://first.test", description: "first fallback" }],
+ news: [{ url: "https://second.test", snippet: "second fallback" }],
+ } as any;
+
+ const result = await applySearchHighlights(response, "query", logger);
+
+ expect(generateHighlightsBatch).toHaveBeenCalledTimes(1);
+ expect(generateHighlightsBatch).toHaveBeenCalledWith(
+ "query",
+ [
+ {
+ id: "0",
+ markdown: "markdown:<main>index:https://first.test.json</main>",
+ },
+ {
+ id: "1",
+ markdown: "markdown:<main>index:https://second.test.json</main>",
+ },
+ ],
+ { logger },
+ );
+ expect(response.web[0].description).toBe("first highlight");
+ expect(response.news[0].snippet).toBe("second highlight");
+ expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 2 });
+ });
+
+ it("preserves individual fallbacks for missing and empty batch pages", async () => {
+ vi.mocked(generateHighlightsBatch).mockResolvedValue(
+ new Map([["0", { highlights: [], markdown: " " }]]),
+ );
+ const response = {
+ web: [
+ { url: "https://first.test", description: "first fallback" },
+ { url: "https://second.test", description: "second fallback" },
+ ],
+ } as any;
+
+ const result = await applySearchHighlights(response, "query", logger);
+
+ expect(response.web.map((item: any) => item.description)).toEqual([
+ "first fallback",
+ "second fallback",
+ ]);
+ expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 0 });
+ });
+
+ it("preserves every fallback when the batch request fails", async () => {
+ vi.mocked(generateHighlightsBatch).mockResolvedValue(null);
+ const response = {
+ web: [
+ { url: "https://first.test", description: "first fallback" },
+ { url: "https://second.test", description: "second fallback" },
+ ],
+ } as any;
+
+ const result = await applySearchHighlights(response, "query", logger);
+
+ expect(response.web.map((item: any) => item.description)).toEqual([
+ "first fallback",
+ "second fallback",
+ ]);
+ expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 0 });
+ });
+});
diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts
index 2964d1f6e..62a8a0c3b 100644
--- a/apps/api/src/search/highlights.ts
+++ b/apps/api/src/search/highlights.ts
@@ -10,7 +10,7 @@ import { indexGetRecent5 } from "../db/rpc";
import { parseMarkdown } from "../lib/html-to-markdown";
import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";
import type { ScrapeOptions } from "../controllers/v2/types";
-import { generateHighlights } from "./highlight-model";
+import { generateHighlightsBatch } from "./highlight-model";
import { config } from "../config";
// How far back into the index we're willing to reach for highlight source text.
@@ -19,15 +19,11 @@ const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
/**
* Whether the deployment has every dependency the highlights beta needs: the
* index DB (to find cached content), the GCS index bucket (to fetch it), and the
- * highlight model service URL + token (to score it). Missing any => silently
- * skip.
+ * highlight model service URL (to score it). Missing any => silently skip.
*/
export function highlightsEnvReady(): boolean {
return (
- useIndex &&
- !!config.GCS_INDEX_BUCKET_NAME &&
- !!config.HIGHLIGHT_MODEL_URL &&
- !!config.HIGHLIGHT_MODEL_TOKEN
+ useIndex && !!config.GCS_INDEX_BUCKET_NAME && !!config.HIGHLIGHT_MODEL_URL
);
}
@@ -124,9 +120,10 @@ async function getIndexedMarkdownForURL(
* For each search result: look up the URL in our index (last 30 days), and if
* present, replace the provider snippet with query-relevant highlights generated
* from the indexed content. Index lookups run in parallel; each hit's full
- * markdown is sent to the highlight model service, which returns the selected
- * highlights reassembled into a single markdown document. Mutates `response` in
- * place. Results not in the index keep their original snippet.
+ * markdown pages are sent to the highlight model service in one batch, which
+ * returns each page's selected highlights reassembled into a single markdown
+ * document. Mutates `response` in place. Results not in the index keep their
+ * original snippet.
*/
export async function applySearchHighlights(
response: SearchV2Response,
@@ -177,21 +174,27 @@ export async function applySearchHighlights(
});
const indexHits = hits.length;
- // Send each hit's full markdown to the highlight model service in parallel and
- // use the reassembled markdown it returns as the snippet.
+ // Send every hit in one request. IDs are local batch indexes, so a missing or
+ // empty response only falls back the corresponding provider snippet.
let replaced = 0;
if (indexHits > 0) {
- const results = await Promise.all(
- hits.map(h => generateHighlights(query, h.markdown, { logger })),
+ const results = await generateHighlightsBatch(
+ query,
+ hits.map((hit, index) => ({
+ id: String(index),
+ markdown: hit.markdown,
+ })),
+ { logger },
);
- results.forEach((result, i) => {
- if (!result) return;
- const snippet = result.markdown;
- if (snippet.trim() !== "") {
- hits[i].apply(snippet);
- replaced++;
- }
- });
+ if (results) {
+ hits.forEach((hit, index) => {
+ const snippet = results.get(String(index))?.markdown;
+ if (snippet?.trim()) {
+ hit.apply(snippet);
+ replaced++;
+ }
+ });
+ }
}
logger.info("Search highlights applied", {