Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api/search): reimplement highlights beta on query-highlights modal service

mogery063aac86ff5f
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 454c946d5..944ab1647 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -139,8 +139,11 @@ const configSchema = z.object({
CLICKHOUSE_ANALYTICS_URL: z.string().optional(),
CLICKHOUSE_ANALYTICS_DATABASE: z.string().optional(),
- // Search highlights (beta): semantic highlight model endpoint
+ // Search highlights (beta): query-highlights model service. URL is the base
+ // (e.g. https://firecrawl--query-highlights.modal.run); TOKEN is the bearer
+ // token sent on every request.
HIGHLIGHT_MODEL_URL: z.string().optional(),
+ HIGHLIGHT_MODEL_TOKEN: z.string().optional(),
// Fire Engine
FIRE_ENGINE_BETA_URL: z.string().optional(),
diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.ts
new file mode 100644
index 000000000..82cbe6678
--- /dev/null
+++ b/apps/api/src/search/highlight-model.test.ts
@@ -0,0 +1,128 @@
+vi.mock("../config", () => ({
+ config: {
+ HIGHLIGHT_MODEL_URL: "https://highlight.test",
+ HIGHLIGHT_MODEL_TOKEN: "secret-token",
+ },
+}));
+
+import { generateHighlightsBatch } from "./highlight-model";
+
+const logger = {
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ debug: vi.fn(),
+} as any;
+
+function mockFetchOnce(body: unknown, ok = true, status = 200) {
+ const fetchMock = vi.fn().mockResolvedValue({
+ ok,
+ status,
+ json: async () => body,
+ text: async () => JSON.stringify(body),
+ });
+ vi.stubGlobal("fetch", fetchMock);
+ return fetchMock;
+}
+
+afterEach(() => {
+ vi.unstubAllGlobals();
+ vi.clearAllMocks();
+});
+
+describe("generateHighlightsBatch", () => {
+ it("returns [] without calling the service for an empty batch", async () => {
+ const fetchMock = mockFetchOnce({ results: [] });
+ const out = await generateHighlightsBatch([], { logger });
+ expect(out).toEqual([]);
+ expect(fetchMock).not.toHaveBeenCalled();
+ });
+
+ it("posts every page to /batch_highlight with the bearer token", async () => {
+ const fetchMock = mockFetchOnce({
+ results: [
+ { pruned_markdown: "Highlight A\n" },
+ { pruned_markdown: "Highlight B\n" },
+ ],
+ });
+
+ await generateHighlightsBatch(
+ [
+ { query: "q1", markdown: "page one" },
+ { query: "q2", markdown: "page two" },
+ ],
+ { logger },
+ );
+
+ expect(fetchMock).toHaveBeenCalledTimes(1);
+ const [url, init] = fetchMock.mock.calls[0];
+ expect(url).toBe("https://highlight.test/batch_highlight");
+ expect(init.method).toBe("POST");
+ expect(init.headers.Authorization).toBe("Bearer secret-token");
+
+ const sent = JSON.parse(init.body);
+ expect(sent.requests).toHaveLength(2);
+ expect(sent.requests[0]).toMatchObject({ query: "q1", markdown: "page one" });
+ // Threshold + budget are always sent so the cutoff matches the trained format.
+ expect(typeof sent.requests[0].threshold).toBe("number");
+ expect(typeof sent.requests[0].max_highlight_chars).toBe("number");
+ });
+
+ it("maps pruned_markdown back to each item, trimmed, aligned by index", async () => {
+ mockFetchOnce({
+ results: [
+ { pruned_markdown: "First page highlight\n" },
+ { pruned_markdown: "Second page highlight\nmore\n" },
+ ],
+ });
+
+ const out = await generateHighlightsBatch(
+ [
+ { query: "q1", markdown: "a" },
+ { query: "q2", markdown: "b" },
+ ],
+ { logger },
+ );
+
+ expect(out).toEqual([
+ "First page highlight",
+ "Second page highlight\nmore",
+ ]);
+ });
+
+ it("returns null for empty / missing pruned_markdown entries", async () => {
+ mockFetchOnce({
+ results: [
+ { pruned_markdown: "" },
+ { pruned_markdown: " \n" },
+ {},
+ ],
+ });
+
+ const out = await generateHighlightsBatch(
+ [
+ { query: "q", markdown: "a" },
+ { query: "q", markdown: "b" },
+ { query: "q", markdown: "c" },
+ ],
+ { logger },
+ );
+
+ expect(out).toEqual([null, null, null]);
+ });
+
+ it("falls back to nulls (one per item) when the service errors", async () => {
+ mockFetchOnce({ error: "boom" }, false, 500);
+
+ const out = await generateHighlightsBatch(
+ [
+ { query: "q", markdown: "a" },
+ { query: "q", markdown: "b" },
+ ],
+ { logger },
+ );
+
+ expect(out).toEqual([null, null]);
+ expect(logger.warn).toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.ts
index 817c37f03..e41a192db 100644
--- a/apps/api/src/search/highlight-model.ts
+++ b/apps/api/src/search/highlight-model.ts
@@ -1,114 +1,77 @@
import type { Logger } from "winston";
import { config } from "../config";
-// Semantic highlight model: it scores each sentence of a context against a
-// question (semantic similarity), no LLM. Inference is fast (~10ms), but the
-// model caps context at ~4k tokens, so we chunk the page, score every chunk,
-// pool the per-sentence scores globally, and keep the best sentences. Endpoint
-// is config-gated (HIGHLIGHT_MODEL_URL); callers must confirm it's set via
-// highlightsEnvReady() before invoking.
-
-// ~4 chars/token; 10k chars (~2.5k tokens) leaves headroom under the 4k cap for
-// the question + tokenizer variance (markdown/code tokenizes denser than prose).
-const CHUNK_CHARS = 10000;
-// Bound load for very long pages (chunks run concurrently per result).
-const MAX_CHUNKS = 10;
-// Keep sentences scoring at/above this (matches the model's default threshold).
-const SELECT_THRESHOLD = 0.5;
-// Cap the assembled snippet length.
-const MAX_SELECTED = 12;
+// Query Highlights model service: a query-finetuned span scorer (no LLM). It
+// splits each page's markdown into line-like candidate spans β€” the same shape
+// the model was trained on β€” scores every span against the query, and returns
+// the selected spans concatenated in page order as `pruned_markdown`. The
+// service keeps a resident GPU model, so we send every page through one
+// `/batch_highlight` call instead of one request per page. Endpoint + token are
+// config-gated (HIGHLIGHT_MODEL_URL / HIGHLIGHT_MODEL_TOKEN); callers must
+// confirm they're set via highlightsEnvReady() before invoking.
+
+// Experimental score cutoff for keeping a span, tuned against the line-span
+// format the model was trained on.
+const HIGHLIGHT_THRESHOLD = 0.08;
+// Character budget for the assembled snippet; the service keeps the
+// highest-scoring spans until the budget is reached.
+const MAX_HIGHLIGHT_CHARS = 800;
const REQUEST_TIMEOUT_MS = 30000;
-interface ScoredSentence {
- text: string;
- score: number;
- order: number; // global document order
+interface HighlightItem {
+ query: string;
+ markdown: string;
}
-// Strip markdown syntax so a highlight reads as plain prose: links/images keep
-// their text, bare URLs and emphasis/heading/code marks go, line-number gutters
-// from rendered code blocks are dropped, whitespace collapses.
-function cleanSentence(text: string): string {
- return text
- .replace(/!\[[^\]]*\]\([^)]*\)/g, " ") // images
- .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") // links -> link text
- .replace(/https?:\/\/\S+/g, " ") // bare URLs
- .replace(/^\s*\d+\s*/, "") // leading code-block line number
- .replace(/[#*`>_~|\\]+/g, " ") // markdown marks
- .replace(/\s+/g, " ")
- .trim();
+interface HighlightResult {
+ pruned_markdown?: string;
}
-// Drop markdown/code artifacts (line numbers, "=====", "//", "[ 01 / 06 ]")
-// that the semantic model sometimes scores highly. Real sentences have prose.
-function isContentful(text: string): boolean {
- const letters = (text.match(/[a-zA-Z]/g) ?? []).length;
- return letters >= 15;
+interface BatchHighlightResponse {
+ results?: HighlightResult[];
}
-interface HighlightModelResponse {
- kept_sentences?: string[];
- sentence_probabilities?: number[];
+// The service returns the selected spans joined with "\n" (often with a trailing
+// newline). Trim to a clean snippet; null when nothing was selected.
+function prunedToSnippet(result: HighlightResult | undefined): string | null {
+ const pruned = result?.pruned_markdown;
+ if (typeof pruned !== "string") return null;
+ const trimmed = pruned.trim();
+ return trimmed === "" ? null : trimmed;
}
/**
- * Split markdown into chunks under the model's token budget. Splits on blank
- * lines (paragraph boundaries) to avoid cutting mid-sentence; oversized
- * paragraphs are hard-split as a last resort. Capped at MAX_CHUNKS.
+ * Generate query-relevant highlights for many pages in a single batch call to
+ * the query-highlights model. Each page's markdown is sent as-is; the service
+ * splits it into the line-like spans used in training, scores them against the
+ * query, and returns the selected spans joined in page order
+ * (`pruned_markdown`). Returns an array aligned to `items`: the highlight
+ * snippet for each page, or null when nothing clears the threshold (or the whole
+ * batch fails β€” highlights are best-effort, callers keep the provider snippet).
*/
-function chunkMarkdown(markdown: string): string[] {
- const chunks: string[] = [];
- let current = "";
-
- const flush = () => {
- if (current.trim() !== "") chunks.push(current);
- current = "";
- };
-
- for (const para of markdown.split(/\n{2,}/)) {
- if (chunks.length >= MAX_CHUNKS) break;
-
- if (para.length > CHUNK_CHARS) {
- flush();
- for (
- let i = 0;
- i < para.length && chunks.length < MAX_CHUNKS;
- i += CHUNK_CHARS
- ) {
- chunks.push(para.slice(i, i + CHUNK_CHARS));
- }
- continue;
- }
-
- if (current.length + para.length + 2 > CHUNK_CHARS) {
- flush();
- }
- current += (current === "" ? "" : "\n\n") + para;
- }
-
- if (chunks.length < MAX_CHUNKS) flush();
- return chunks.slice(0, MAX_CHUNKS);
-}
+export async function generateHighlightsBatch(
+ items: HighlightItem[],
+ opts: { logger: Logger },
+): Promise<(string | null)[]> {
+ if (items.length === 0) return [];
-async function scoreChunk(
- question: string,
- context: string,
- logger: Logger,
-): Promise<{ text: string; score: number }[]> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+ const start = Date.now();
try {
- const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/v1/highlight`, {
+ const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/batch_highlight`, {
method: "POST",
- headers: { "Content-Type": "application/json" },
+ headers: {
+ "Content-Type": "application/json",
+ Authorization: `Bearer ${config.HIGHLIGHT_MODEL_TOKEN}`,
+ },
body: JSON.stringify({
- question,
- context,
- // threshold 0 => every sentence is returned with its score, so we can
- // pool scores across chunks and select globally.
- threshold: 0.0,
- language: "auto",
- return_sentence_metrics: true,
+ requests: items.map(item => ({
+ query: item.query,
+ markdown: item.markdown,
+ threshold: HIGHLIGHT_THRESHOLD,
+ max_highlight_chars: MAX_HIGHLIGHT_CHARS,
+ })),
}),
signal: controller.signal,
});
@@ -120,84 +83,24 @@ async function scoreChunk(
);
}
- const data = (await res.json()) as HighlightModelResponse;
- const sentences = data.kept_sentences ?? [];
- const probs = data.sentence_probabilities ?? [];
- const n = Math.min(sentences.length, probs.length);
- const out: { text: string; score: number }[] = [];
- for (let i = 0; i < n; i++) {
- out.push({ text: sentences[i], score: probs[i] });
- }
- return out;
- } finally {
- clearTimeout(timer);
- }
-}
-
-/**
- * Generate query-relevant highlights from page markdown using the semantic
- * highlight model. Returns the assembled highlight string (best sentences in
- * document order), or null if nothing clears the threshold / all chunks fail.
- */
-export async function generateSemanticHighlights(
- markdown: string,
- query: string,
- opts: { logger: Logger },
-): Promise<string | null> {
- const chunks = chunkMarkdown(markdown);
- if (chunks.length === 0) return null;
+ const data = (await res.json()) as BatchHighlightResponse;
+ const results = data.results ?? [];
+ const snippets = items.map((_, i) => prunedToSnippet(results[i]));
- const start = Date.now();
- const perChunk = await Promise.all(
- chunks.map(async (chunk, idx) => {
- try {
- return await scoreChunk(query, chunk, opts.logger);
- } catch (error) {
- opts.logger.warn("highlight model chunk failed", {
- error: error instanceof Error ? error.message : String(error),
- chunkIdx: idx,
- });
- return [];
- }
- }),
- );
+ opts.logger.info("query highlights batch generated", {
+ pages: items.length,
+ withHighlights: snippets.filter(Boolean).length,
+ elapsedMs: Date.now() - start,
+ });
- // Pool every scored sentence, preserving document order (Promise.all keeps
- // chunk order, and sentences within a chunk are already in order).
- const all: ScoredSentence[] = [];
- let order = 0;
- for (const chunkSentences of perChunk) {
- for (const s of chunkSentences) {
- const idx = order++;
- const cleaned = cleanSentence(s.text);
- if (isContentful(cleaned)) {
- all.push({ text: cleaned, score: s.score, order: idx });
- }
- }
+ return snippets;
+ } catch (error) {
+ opts.logger.warn("query highlights batch failed", {
+ error: error instanceof Error ? error.message : String(error),
+ pages: items.length,
+ });
+ return items.map(() => null);
+ } finally {
+ clearTimeout(timer);
}
- if (all.length === 0) return null;
-
- // Pick the best: keep sentences at/above the threshold, cap to the top
- // MAX_SELECTED by score, then restore document order for readability.
- let selected = all.filter(s => s.score >= SELECT_THRESHOLD);
- if (selected.length === 0) return null;
- selected.sort((a, b) => b.score - a.score);
- selected = selected.slice(0, MAX_SELECTED);
- selected.sort((a, b) => a.order - b.order);
-
- const text = selected
- .map(s => s.text.trim())
- .join(" ")
- .replace(/\s+/g, " ")
- .trim();
-
- opts.logger.info("semantic highlights generated", {
- chunks: chunks.length,
- scoredSentences: all.length,
- selected: selected.length,
- topScore: all.reduce((m, s) => Math.max(m, s.score), 0),
- elapsedMs: Date.now() - start,
- });
-
- return text === "" ? null : text;
}
diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts
index 4bb4f44c3..9a6eb0a9f 100644
--- a/apps/api/src/search/highlights.ts
+++ b/apps/api/src/search/highlights.ts
@@ -10,7 +10,7 @@ import { indexGetRecent5 } from "../db/rpc";
import { parseMarkdown } from "../lib/html-to-markdown";
import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";
import type { ScrapeOptions } from "../controllers/v2/types";
-import { generateSemanticHighlights } from "./highlight-model";
+import { generateHighlightsBatch } from "./highlight-model";
import { config } from "../config";
// How far back into the index we're willing to reach for highlight source text.
@@ -19,11 +19,15 @@ const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
/**
* Whether the deployment has every dependency the highlights beta needs: the
* index DB (to find cached content), the GCS index bucket (to fetch it), and the
- * highlight model endpoint (to score it). Missing any => silently skip.
+ * highlight model service URL + token (to score it). Missing any => silently
+ * skip.
*/
export function highlightsEnvReady(): boolean {
return (
- useIndex && !!config.GCS_INDEX_BUCKET_NAME && !!config.HIGHLIGHT_MODEL_URL
+ useIndex &&
+ !!config.GCS_INDEX_BUCKET_NAME &&
+ !!config.HIGHLIGHT_MODEL_URL &&
+ !!config.HIGHLIGHT_MODEL_TOKEN
);
}
@@ -115,89 +119,78 @@ async function getIndexedMarkdownForURL(
}
}
-async function highlightOne(
- url: string,
- query: string,
- logger: Logger,
- apply: (highlights: string) => void,
- counters: { indexHits: number; replaced: number },
-): Promise<void> {
- const markdown = await getIndexedMarkdownForURL(url, logger);
- if (!markdown) {
- return;
- }
- counters.indexHits++;
-
- const highlights = await generateSemanticHighlights(markdown, query, {
- logger,
- });
- if (highlights && highlights.trim() !== "") {
- apply(highlights);
- counters.replaced++;
- }
-}
-
/**
- * For each search result, in parallel: look up the URL in our index (last 30
- * days), and if present, replace the provider snippet with query-relevant
- * highlights generated from the indexed content. Mutates `response` in place.
- * Results not in the index keep their original snippet.
+ * For each search result: look up the URL in our index (last 30 days), and if
+ * present, replace the provider snippet with query-relevant highlights generated
+ * from the indexed content. Index lookups run in parallel, then every hit is
+ * scored in a single batch call to the query-highlights model. Mutates
+ * `response` in place. Results not in the index keep their original snippet.
*/
export async function applySearchHighlights(
response: SearchV2Response,
query: string,
logger: Logger,
): Promise<{ attempted: number; indexHits: number; replaced: number }> {
- const counters = { indexHits: 0, replaced: 0 };
- const tasks: Promise<void>[] = [];
const start = Date.now();
- // Web results carry the snippet in `description` β€” replace it in place.
+ // Collect every result we could highlight, each with a setter for its snippet
+ // field: web results carry it in `description`, news results in `snippet`.
+ const targets: { url: string; apply: (h: string) => void }[] = [];
for (const result of response.web ?? []) {
if (!result.url) continue;
- tasks.push(
- highlightOne(
- result.url,
- query,
- logger,
- h => {
- result.description = h;
- },
- counters,
- ),
- );
+ targets.push({
+ url: result.url,
+ apply: h => {
+ result.description = h;
+ },
+ });
}
-
- // News results carry the snippet in `snippet` β€” replace it in place.
for (const result of response.news ?? []) {
if (!result.url) continue;
- const url = result.url;
- tasks.push(
- highlightOne(
- url,
- query,
- logger,
- h => {
- result.snippet = h;
- },
- counters,
- ),
- );
+ targets.push({
+ url: result.url,
+ apply: h => {
+ result.snippet = h;
+ },
+ });
}
- const attempted = tasks.length;
- await Promise.all(tasks);
+ const attempted = targets.length;
+ if (attempted === 0) {
+ return { attempted, indexHits: 0, replaced: 0 };
+ }
+
+ // Look up indexed markdown for every URL in parallel, keeping only the hits.
+ const markdowns = await Promise.all(
+ targets.map(t => getIndexedMarkdownForURL(t.url, logger)),
+ );
+ const hits: { apply: (h: string) => void; markdown: string }[] = [];
+ markdowns.forEach((markdown, i) => {
+ if (markdown) hits.push({ apply: targets[i].apply, markdown });
+ });
+ const indexHits = hits.length;
+
+ // Score every hit in one batch call, then apply the highlights that came back.
+ let replaced = 0;
+ if (indexHits > 0) {
+ const snippets = await generateHighlightsBatch(
+ hits.map(h => ({ query, markdown: h.markdown })),
+ { logger },
+ );
+ snippets.forEach((snippet, i) => {
+ if (snippet && snippet.trim() !== "") {
+ hits[i].apply(snippet);
+ replaced++;
+ }
+ });
+ }
logger.info("Search highlights applied", {
attempted,
- indexHits: counters.indexHits,
- replaced: counters.replaced,
+ indexHits,
+ replaced,
timeTakenMs: Date.now() - start,
});
- return {
- attempted,
- indexHits: counters.indexHits,
- replaced: counters.replaced,
- };
+ return { attempted, indexHits, replaced };
}