Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): batch query highlights (#3993)
Gergő Móriczb113bec5cc78
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 35c082082..8a672944d 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -159,9 +159,8 @@ const configSchema = z.object({CLICKHOUSE_ANALYTICS_URL: z.string().optional(),CLICKHOUSE_ANALYTICS_DATABASE: z.string().optional(),- // Search highlights (beta): query-highlights model service. URL is the base- // (e.g. https://firecrawl--query-highlights.modal.run); TOKEN is the bearer- // token sent on every request.+ // Search highlights (beta): highlighter service base URL. TOKEN is optional+ // bearer auth for legacy/external services; the in-cluster service omits it.HIGHLIGHT_MODEL_URL: z.string().optional(),HIGHLIGHT_MODEL_TOKEN: z.string().optional(),diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.tsindex 23cc19ba4..b70925d8a 100644--- a/apps/api/src/search/highlight-model.test.ts+++ b/apps/api/src/search/highlight-model.test.ts@@ -5,7 +5,8 @@ vi.mock("../config", () => ({},}));-import { generateHighlights } from "./highlight-model";+import { generateHighlightsBatch } from "./highlight-model";+import { config } from "../config";const logger = {info: vi.fn(),@@ -30,34 +31,81 @@ afterEach(() => {vi.clearAllMocks();});-describe("generateHighlights", () => {- it("posts the query and full markdown to /highlight with the bearer token", async () => {- const fetchMock = mockFetchOnce({ highlights: [], markdown: "" });+describe("generateHighlightsBatch", () => {+ it("posts every page to one /batch_highlight call with the bearer token", async () => {+ const fetchMock = mockFetchOnce({ pages: [] });- await generateHighlights("q1", "# Page\n\nbody text", { logger });+ await generateHighlightsBatch(+ "q1",+ [+ { id: "0", markdown: "# First\n\nbody text" },+ { id: "1", markdown: "# Second\n\nmore text" },+ ],+ { logger },+ );expect(fetchMock).toHaveBeenCalledTimes(1);const [url, init] = fetchMock.mock.calls[0];- expect(url).toBe("https://highlight.test/highlight");+ expect(url).toBe("https://highlight.test/batch_highlight");expect(init.method).toBe("POST");expect(init.headers.Authorization).toBe("Bearer secret-token");const sent = JSON.parse(init.body);- expect(sent).toEqual({ query: "q1", markdown: "# Page\n\nbody text" });+ expect(sent).toEqual({+ query: "q1",+ pages: [+ { id: "0", markdown: "# First\n\nbody text" },+ { id: "1", markdown: "# Second\n\nmore text" },+ ],+ });});- it("returns the highlights and reassembled markdown from the service", async () => {+ it("omits bearer auth for the in-cluster service when no token is configured", async () => {+ const token = config.HIGHLIGHT_MODEL_TOKEN;+ config.HIGHLIGHT_MODEL_TOKEN = undefined;+ const fetchMock = mockFetchOnce({ pages: [] });++ try {+ await generateHighlightsBatch("q1", [{ id: "0", markdown: "markdown" }], {+ logger,+ });+ } finally {+ config.HIGHLIGHT_MODEL_TOKEN = token;+ }++ const [, init] = fetchMock.mock.calls[0];+ expect(init.headers).toEqual({ "Content-Type": "application/json" });+ });++ it("returns successful pages keyed by ID and leaves missing pages absent", async () => {mockFetchOnce({- highlights: [- { block_index: 0, score: 0.9 },- { block_index: 2, score: 0.5 },+ pages: [+ {+ id: "1",+ cache: "hit",+ output: {+ highlights: [+ { block_index: 0, score: 0.9 },+ { block_index: 2, score: 0.5 },+ ],+ markdown: "reassembled answer",+ },+ },+ { id: "invalid-without-output" },],- markdown: "reassembled answer",});- const out = await generateHighlights("q", "some markdown", { logger });+ const out = await generateHighlightsBatch(+ "q",+ [+ { id: "0", markdown: "missing" },+ { id: "1", markdown: "some markdown" },+ ],+ { logger },+ );- expect(out).toEqual({+ expect(out?.get("0")).toBeUndefined();+ expect(out?.get("1")).toEqual({highlights: [{ block_index: 0, score: 0.9 },{ block_index: 2, score: 0.5 },@@ -66,37 +114,117 @@ describe("generateHighlights", () => {});});- it("debug-logs the highlights array", async () => {+ it("debug-logs batch coverage and highlights", async () => {mockFetchOnce({- highlights: [{ block_index: 1, score: 0.8 }],- markdown: "x",+ pages: [+ {+ id: "page-1",+ output: {+ highlights: [{ block_index: 1, score: 0.8 }],+ markdown: "x",+ },+ },+ ],});- await generateHighlights("q", "md", { logger });+ await generateHighlightsBatch("q", [{ id: "page-1", markdown: "md" }], {+ logger,+ });expect(logger.debug).toHaveBeenCalledWith(- "query highlights",+ "query highlights batch",expect.objectContaining({canonicalLog: "search/highlights",- highlights: [{ block_index: 1, score: 0.8 }],+ requestedPages: 1,+ returnedPages: 1,+ pages: [+ {+ id: "page-1",+ highlights: [{ block_index: 1, score: 0.8 }],+ },+ ],}),);});- it("defaults missing fields to empty highlights and markdown", async () => {+ it("defaults missing pages to an empty result map", async () => {mockFetchOnce({});- const out = await generateHighlights("q", "md", { logger });+ const out = await generateHighlightsBatch(+ "q",+ [{ id: "0", markdown: "md" }],+ { logger },+ );- expect(out).toEqual({ highlights: [], markdown: "" });+ expect(out).toEqual(new Map());+ });++ it("does not call the service for an empty batch", async () => {+ const fetchMock = vi.fn();+ vi.stubGlobal("fetch", fetchMock);++ const out = await generateHighlightsBatch("q", [], { logger });++ expect(fetchMock).not.toHaveBeenCalled();+ expect(out).toEqual(new Map());});it("returns null when the service errors", async () => {mockFetchOnce({ error: "boom" }, false, 500);- const out = await generateHighlights("q", "md", { logger });+ const out = await generateHighlightsBatch(+ "q",+ [{ id: "0", markdown: "md" }],+ { logger },+ );expect(out).toBeNull();expect(logger.warn).toHaveBeenCalled();});++ it("falls back to legacy per-page calls while the old service URL is configured", async () => {+ const fetchMock = vi+ .fn()+ .mockResolvedValueOnce({+ ok: false,+ status: 400,+ text: async () => "legacy batch contract",+ })+ .mockResolvedValueOnce({+ ok: true,+ status: 200,+ json: async () => ({ markdown: "first legacy highlight" }),+ text: async () => "",+ })+ .mockResolvedValueOnce({+ ok: false,+ status: 500,+ json: async () => ({}),+ text: async () => "page failed",+ });+ vi.stubGlobal("fetch", fetchMock);++ const out = await generateHighlightsBatch(+ "q",+ [+ { id: "0", markdown: "first" },+ { id: "1", markdown: "second" },+ ],+ { logger },+ );++ expect(fetchMock).toHaveBeenCalledTimes(3);+ expect(fetchMock.mock.calls.map(call => call[0])).toEqual([+ "https://highlight.test/batch_highlight",+ "https://highlight.test/highlight",+ "https://highlight.test/highlight",+ ]);+ expect(out).toEqual(+ new Map([["0", { highlights: [], markdown: "first legacy highlight" }]]),+ );+ expect(logger.warn).toHaveBeenCalledWith(+ "legacy query highlight failed",+ expect.objectContaining({ pageId: "1" }),+ );+ });});diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.tsindex 427c9543a..1feb53c09 100644--- a/apps/api/src/search/highlight-model.ts+++ b/apps/api/src/search/highlight-model.ts@@ -1,12 +1,13 @@import type { Logger } from "winston";import { config } from "../config";-// Query Highlights model service: given a page's full markdown and a query, the-// service returns the query-relevant highlights plus those highlights-// reassembled into a single markdown document (in document order). We push each-// page through one `/highlight` call. Endpoint + token are config-gated-// (HIGHLIGHT_MODEL_URL / HIGHLIGHT_MODEL_TOKEN); callers must confirm they're-// set via highlightsEnvReady() before invoking.+// Query Highlights model service: given full markdown pages and a query, the+// service returns query-relevant highlights plus each page's highlights+// reassembled into a single markdown document (in document order). All indexed+// search results go through one `/batch_highlight` call. The endpoint is+// URL-config-gated; callers must confirm it is set via highlightsEnvReady()+// before invoking. Bearer auth remains optional for legacy/external services;+// the in-cluster GCP Stage 1 service relies on cluster network isolation.const REQUEST_TIMEOUT_MS = 30000;@@ -25,58 +26,111 @@ interface HighlightResponse {markdown?: string;}-// Result of one highlight call: the service's highlight entries plus the+interface HighlightBatchPageResponse {+ id?: string;+ output?: HighlightResponse;+}++interface HighlightBatchResponse {+ pages?: HighlightBatchPageResponse[];+}++interface HighlightPage {+ id: string;+ markdown: string;+}++// Result for one page in the batch: the service's highlight entries plus the// reassembled markdown document.interface HighlightResult {highlights: Highlight[];markdown: string;}+function requestHeaders(): Record<string, string> {+ const headers: Record<string, string> = {+ "Content-Type": "application/json",+ };+ if (config.HIGHLIGHT_MODEL_TOKEN) {+ headers.Authorization = `Bearer ${config.HIGHLIGHT_MODEL_TOKEN}`;+ }+ return headers;+}+/**- * Generate query highlights for one page by sending its full markdown and the- * query to the highlight model service. Returns the service's highlight entries- * plus the reassembled markdown, or null when the call fails (so callers can- * keep the provider snippet).+ * Generate query highlights for every indexed result in one request. Results+ * are keyed by the caller-provided page ID so a missing page can fall back to+ * its provider snippet without discarding successful pages. Returns null when+ * the whole call fails.*/-export async function generateHighlights(+export async function generateHighlightsBatch(query: string,- markdown: string,+ pages: HighlightPage[],opts: { logger: Logger },-): Promise<HighlightResult | null> {+): Promise<Map<string, HighlightResult> | null> {+ if (pages.length === 0) {+ return new Map();+ }+const controller = new AbortController();const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);const start = Date.now();try {- const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/highlight`, {+ const baseUrl = config.HIGHLIGHT_MODEL_URL!.replace(/\/$/, "");+ const res = await fetch(`${baseUrl}/batch_highlight`, {method: "POST",- headers: {- "Content-Type": "application/json",- Authorization: `Bearer ${config.HIGHLIGHT_MODEL_TOKEN}`,- },- body: JSON.stringify({ query, markdown }),+ headers: requestHeaders(),+ body: JSON.stringify({ query, pages }),signal: controller.signal,});if (!res.ok) {const body = await res.text().catch(() => "");+ // During rollout the configured service may still be the legacy Modal+ // endpoint, whose /batch_highlight contract uses {requests: [...]}. Keep+ // highlights available until infra switches the URL to GCP Stage 1.+ if (res.status === 400 || res.status === 404) {+ opts.logger.info("query highlights using legacy per-page fallback", {+ canonicalLog: "search/highlights",+ status: res.status,+ });+ return await generateLegacyHighlightsBatch(+ baseUrl,+ query,+ pages,+ controller.signal,+ opts,+ );+ }throw new Error(`highlight model HTTP ${res.status}: ${body.slice(0, 200)}`,);}- const data = (await res.json()) as HighlightResponse;- const highlights = data.highlights ?? [];+ const data = (await res.json()) as HighlightBatchResponse;+ const results = new Map<string, HighlightResult>();+ for (const page of data.pages ?? []) {+ if (typeof page.id !== "string" || !page.output) continue;+ results.set(page.id, {+ highlights: page.output.highlights ?? [],+ markdown: page.output.markdown ?? "",+ });+ }- // Canonical log of the highlights array, debug-level only.- opts.logger.debug("query highlights", {+ opts.logger.debug("query highlights batch", {canonicalLog: "search/highlights",- highlights,+ requestedPages: pages.length,+ returnedPages: results.size,+ pages: Array.from(results, ([id, result]) => ({+ id,+ highlights: result.highlights,+ })),elapsedMs: Date.now() - start,});- return { highlights, markdown: data.markdown ?? "" };+ return results;} catch (error) {- opts.logger.warn("query highlights failed", {+ opts.logger.warn("query highlights batch failed", {canonicalLog: "search/highlights",error: error instanceof Error ? error.message : String(error),});@@ -85,3 +139,46 @@ export async function generateHighlights(clearTimeout(timer);}}++async function generateLegacyHighlightsBatch(+ baseUrl: string,+ query: string,+ pages: HighlightPage[],+ signal: AbortSignal,+ opts: { logger: Logger },+): Promise<Map<string, HighlightResult>> {+ const entries = await Promise.all(+ pages.map(async page => {+ try {+ const res = await fetch(`${baseUrl}/highlight`, {+ method: "POST",+ headers: requestHeaders(),+ body: JSON.stringify({ query, markdown: page.markdown }),+ signal,+ });+ if (!res.ok) {+ const body = await res.text().catch(() => "");+ throw new Error(+ `highlight model HTTP ${res.status}: ${body.slice(0, 200)}`,+ );+ }+ const data = (await res.json()) as HighlightResponse;+ return [+ page.id,+ {+ highlights: data.highlights ?? [],+ markdown: data.markdown ?? "",+ },+ ] as const;+ } catch (error) {+ opts.logger.warn("legacy query highlight failed", {+ canonicalLog: "search/highlights",+ pageId: page.id,+ error: error instanceof Error ? error.message : String(error),+ });+ return null;+ }+ }),+ );+ return new Map(entries.filter(entry => entry !== null));+}diff --git a/apps/api/src/search/highlights.test.ts b/apps/api/src/search/highlights.test.tsnew file mode 100644index 000000000..ccaa8791f--- /dev/null+++ b/apps/api/src/search/highlights.test.ts@@ -0,0 +1,141 @@+vi.mock("../config", () => ({+ config: {+ GCS_INDEX_BUCKET_NAME: "index-bucket",+ HIGHLIGHT_MODEL_URL: "https://highlight.test",+ HIGHLIGHT_MODEL_TOKEN: "secret-token",+ },+}));++vi.mock("../services", () => ({+ useIndex: true,+ normalizeURLForIndex: vi.fn((url: string) => url),+ hashURL: vi.fn((url: string) => url),+ getIndexFromGCS: vi.fn(async (key: string) => ({+ html: `<main>${key}</main>`,+ })),+}));++vi.mock("../db/rpc", () => ({+ indexGetRecent5: vi.fn(async ({ url_hash }: { url_hash: string }) => [+ {+ id: `index:${url_hash}`,+ status: 200,+ created_at: new Date("2026-07-11T00:00:00Z"),+ },+ ]),+}));++vi.mock("../lib/html-to-markdown", () => ({+ parseMarkdown: vi.fn(async (html: string) => `markdown:${html}`),+}));++vi.mock("../scraper/scrapeURL/lib/removeUnwantedElements", () => ({+ htmlTransform: vi.fn(async (html: string) => html),+}));++vi.mock("./highlight-model", () => ({+ generateHighlightsBatch: vi.fn(),+}));++import { generateHighlightsBatch } from "./highlight-model";+import { config } from "../config";+import { applySearchHighlights, highlightsEnvReady } from "./highlights";++const logger = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+ child: vi.fn(function () {+ return this;+ }),+} as any;++afterEach(() => {+ vi.clearAllMocks();+});++describe("applySearchHighlights", () => {+ it("enables the in-cluster service without requiring a bearer token", () => {+ const token = config.HIGHLIGHT_MODEL_TOKEN;+ config.HIGHLIGHT_MODEL_TOKEN = undefined;++ try {+ expect(highlightsEnvReady()).toBe(true);+ } finally {+ config.HIGHLIGHT_MODEL_TOKEN = token;+ }+ });++ it("sends indexed web and news results in one batch and applies responses by ID", async () => {+ vi.mocked(generateHighlightsBatch).mockResolvedValue(+ new Map([+ ["0", { highlights: [], markdown: "first highlight" }],+ ["1", { highlights: [], markdown: "second highlight" }],+ ]),+ );+ const response = {+ web: [{ url: "https://first.test", description: "first fallback" }],+ news: [{ url: "https://second.test", snippet: "second fallback" }],+ } as any;++ const result = await applySearchHighlights(response, "query", logger);++ expect(generateHighlightsBatch).toHaveBeenCalledTimes(1);+ expect(generateHighlightsBatch).toHaveBeenCalledWith(+ "query",+ [+ {+ id: "0",+ markdown: "markdown:<main>index:https://first.test.json</main>",+ },+ {+ id: "1",+ markdown: "markdown:<main>index:https://second.test.json</main>",+ },+ ],+ { logger },+ );+ expect(response.web[0].description).toBe("first highlight");+ expect(response.news[0].snippet).toBe("second highlight");+ expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 2 });+ });++ it("preserves individual fallbacks for missing and empty batch pages", async () => {+ vi.mocked(generateHighlightsBatch).mockResolvedValue(+ new Map([["0", { highlights: [], markdown: " " }]]),+ );+ const response = {+ web: [+ { url: "https://first.test", description: "first fallback" },+ { url: "https://second.test", description: "second fallback" },+ ],+ } as any;++ const result = await applySearchHighlights(response, "query", logger);++ expect(response.web.map((item: any) => item.description)).toEqual([+ "first fallback",+ "second fallback",+ ]);+ expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 0 });+ });++ it("preserves every fallback when the batch request fails", async () => {+ vi.mocked(generateHighlightsBatch).mockResolvedValue(null);+ const response = {+ web: [+ { url: "https://first.test", description: "first fallback" },+ { url: "https://second.test", description: "second fallback" },+ ],+ } as any;++ const result = await applySearchHighlights(response, "query", logger);++ expect(response.web.map((item: any) => item.description)).toEqual([+ "first fallback",+ "second fallback",+ ]);+ expect(result).toEqual({ attempted: 2, indexHits: 2, replaced: 0 });+ });+});diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.tsindex 2964d1f6e..62a8a0c3b 100644--- a/apps/api/src/search/highlights.ts+++ b/apps/api/src/search/highlights.ts@@ -10,7 +10,7 @@ import { indexGetRecent5 } from "../db/rpc";import { parseMarkdown } from "../lib/html-to-markdown";import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";import type { ScrapeOptions } from "../controllers/v2/types";-import { generateHighlights } from "./highlight-model";+import { generateHighlightsBatch } from "./highlight-model";import { config } from "../config";// How far back into the index we're willing to reach for highlight source text.@@ -19,15 +19,11 @@ const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days/*** Whether the deployment has every dependency the highlights beta needs: the* index DB (to find cached content), the GCS index bucket (to fetch it), and the- * highlight model service URL + token (to score it). Missing any => silently- * skip.+ * highlight model service URL (to score it). Missing any => silently skip.*/export function highlightsEnvReady(): boolean {return (- useIndex &&- !!config.GCS_INDEX_BUCKET_NAME &&- !!config.HIGHLIGHT_MODEL_URL &&- !!config.HIGHLIGHT_MODEL_TOKEN+ useIndex && !!config.GCS_INDEX_BUCKET_NAME && !!config.HIGHLIGHT_MODEL_URL);}@@ -124,9 +120,10 @@ async function getIndexedMarkdownForURL(* For each search result: look up the URL in our index (last 30 days), and if* present, replace the provider snippet with query-relevant highlights generated* from the indexed content. Index lookups run in parallel; each hit's full- * markdown is sent to the highlight model service, which returns the selected- * highlights reassembled into a single markdown document. Mutates `response` in- * place. Results not in the index keep their original snippet.+ * markdown pages are sent to the highlight model service in one batch, which+ * returns each page's selected highlights reassembled into a single markdown+ * document. Mutates `response` in place. Results not in the index keep their+ * original snippet.*/export async function applySearchHighlights(response: SearchV2Response,@@ -177,21 +174,27 @@ export async function applySearchHighlights(});const indexHits = hits.length;- // Send each hit's full markdown to the highlight model service in parallel and- // use the reassembled markdown it returns as the snippet.+ // Send every hit in one request. IDs are local batch indexes, so a missing or+ // empty response only falls back the corresponding provider snippet.let replaced = 0;if (indexHits > 0) {- const results = await Promise.all(- hits.map(h => generateHighlights(query, h.markdown, { logger })),+ const results = await generateHighlightsBatch(+ query,+ hits.map((hit, index) => ({+ id: String(index),+ markdown: hit.markdown,+ })),+ { logger },);- results.forEach((result, i) => {- if (!result) return;- const snippet = result.markdown;- if (snippet.trim() !== "") {- hits[i].apply(snippet);- replaced++;- }- });+ if (results) {+ hits.forEach((hit, index) => {+ const snippet = results.get(String(index))?.markdown;+ if (snippet?.trim()) {+ hit.apply(snippet);+ replaced++;+ }+ });+ }}logger.info("Search highlights applied", {