Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/search): reimplement highlights beta on query-highlights modal service
mogery063aac86ff5f
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 454c946d5..944ab1647 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -139,8 +139,11 @@ const configSchema = z.object({CLICKHOUSE_ANALYTICS_URL: z.string().optional(),CLICKHOUSE_ANALYTICS_DATABASE: z.string().optional(),- // Search highlights (beta): semantic highlight model endpoint+ // Search highlights (beta): query-highlights model service. URL is the base+ // (e.g. https://firecrawl--query-highlights.modal.run); TOKEN is the bearer+ // token sent on every request.HIGHLIGHT_MODEL_URL: z.string().optional(),+ HIGHLIGHT_MODEL_TOKEN: z.string().optional(),// Fire EngineFIRE_ENGINE_BETA_URL: z.string().optional(),diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.tsnew file mode 100644index 000000000..82cbe6678--- /dev/null+++ b/apps/api/src/search/highlight-model.test.ts@@ -0,0 +1,128 @@+vi.mock("../config", () => ({+ config: {+ HIGHLIGHT_MODEL_URL: "https://highlight.test",+ HIGHLIGHT_MODEL_TOKEN: "secret-token",+ },+}));++import { generateHighlightsBatch } from "./highlight-model";++const logger = {+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ debug: vi.fn(),+} as any;++function mockFetchOnce(body: unknown, ok = true, status = 200) {+ const fetchMock = vi.fn().mockResolvedValue({+ ok,+ status,+ json: async () => body,+ text: async () => JSON.stringify(body),+ });+ vi.stubGlobal("fetch", fetchMock);+ return fetchMock;+}++afterEach(() => {+ vi.unstubAllGlobals();+ vi.clearAllMocks();+});++describe("generateHighlightsBatch", () => {+ it("returns [] without calling the service for an empty batch", async () => {+ const fetchMock = mockFetchOnce({ results: [] });+ const out = await generateHighlightsBatch([], { logger });+ expect(out).toEqual([]);+ expect(fetchMock).not.toHaveBeenCalled();+ });++ it("posts every page to /batch_highlight with the bearer token", async () => {+ const fetchMock = mockFetchOnce({+ results: [+ { pruned_markdown: "Highlight A\n" },+ { pruned_markdown: "Highlight B\n" },+ ],+ });++ await generateHighlightsBatch(+ [+ { query: "q1", markdown: "page one" },+ { query: "q2", markdown: "page two" },+ ],+ { logger },+ );++ expect(fetchMock).toHaveBeenCalledTimes(1);+ const [url, init] = fetchMock.mock.calls[0];+ expect(url).toBe("https://highlight.test/batch_highlight");+ expect(init.method).toBe("POST");+ expect(init.headers.Authorization).toBe("Bearer secret-token");++ const sent = JSON.parse(init.body);+ expect(sent.requests).toHaveLength(2);+ expect(sent.requests[0]).toMatchObject({ query: "q1", markdown: "page one" });+ // Threshold + budget are always sent so the cutoff matches the trained format.+ expect(typeof sent.requests[0].threshold).toBe("number");+ expect(typeof sent.requests[0].max_highlight_chars).toBe("number");+ });++ it("maps pruned_markdown back to each item, trimmed, aligned by index", async () => {+ mockFetchOnce({+ results: [+ { pruned_markdown: "First page highlight\n" },+ { pruned_markdown: "Second page highlight\nmore\n" },+ ],+ });++ const out = await generateHighlightsBatch(+ [+ { query: "q1", markdown: "a" },+ { query: "q2", markdown: "b" },+ ],+ { logger },+ );++ expect(out).toEqual([+ "First page highlight",+ "Second page highlight\nmore",+ ]);+ });++ it("returns null for empty / missing pruned_markdown entries", async () => {+ mockFetchOnce({+ results: [+ { pruned_markdown: "" },+ { pruned_markdown: " \n" },+ {},+ ],+ });++ const out = await generateHighlightsBatch(+ [+ { query: "q", markdown: "a" },+ { query: "q", markdown: "b" },+ { query: "q", markdown: "c" },+ ],+ { logger },+ );++ expect(out).toEqual([null, null, null]);+ });++ it("falls back to nulls (one per item) when the service errors", async () => {+ mockFetchOnce({ error: "boom" }, false, 500);++ const out = await generateHighlightsBatch(+ [+ { query: "q", markdown: "a" },+ { query: "q", markdown: "b" },+ ],+ { logger },+ );++ expect(out).toEqual([null, null]);+ expect(logger.warn).toHaveBeenCalled();+ });+});diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.tsindex 817c37f03..e41a192db 100644--- a/apps/api/src/search/highlight-model.ts+++ b/apps/api/src/search/highlight-model.ts@@ -1,114 +1,77 @@import type { Logger } from "winston";import { config } from "../config";-// Semantic highlight model: it scores each sentence of a context against a-// question (semantic similarity), no LLM. Inference is fast (~10ms), but the-// model caps context at ~4k tokens, so we chunk the page, score every chunk,-// pool the per-sentence scores globally, and keep the best sentences. Endpoint-// is config-gated (HIGHLIGHT_MODEL_URL); callers must confirm it's set via-// highlightsEnvReady() before invoking.--// ~4 chars/token; 10k chars (~2.5k tokens) leaves headroom under the 4k cap for-// the question + tokenizer variance (markdown/code tokenizes denser than prose).-const CHUNK_CHARS = 10000;-// Bound load for very long pages (chunks run concurrently per result).-const MAX_CHUNKS = 10;-// Keep sentences scoring at/above this (matches the model's default threshold).-const SELECT_THRESHOLD = 0.5;-// Cap the assembled snippet length.-const MAX_SELECTED = 12;+// Query Highlights model service: a query-finetuned span scorer (no LLM). It+// splits each page's markdown into line-like candidate spans β the same shape+// the model was trained on β scores every span against the query, and returns+// the selected spans concatenated in page order as `pruned_markdown`. The+// service keeps a resident GPU model, so we send every page through one+// `/batch_highlight` call instead of one request per page. Endpoint + token are+// config-gated (HIGHLIGHT_MODEL_URL / HIGHLIGHT_MODEL_TOKEN); callers must+// confirm they're set via highlightsEnvReady() before invoking.++// Experimental score cutoff for keeping a span, tuned against the line-span+// format the model was trained on.+const HIGHLIGHT_THRESHOLD = 0.08;+// Character budget for the assembled snippet; the service keeps the+// highest-scoring spans until the budget is reached.+const MAX_HIGHLIGHT_CHARS = 800;const REQUEST_TIMEOUT_MS = 30000;-interface ScoredSentence {- text: string;- score: number;- order: number; // global document order+interface HighlightItem {+ query: string;+ markdown: string;}-// Strip markdown syntax so a highlight reads as plain prose: links/images keep-// their text, bare URLs and emphasis/heading/code marks go, line-number gutters-// from rendered code blocks are dropped, whitespace collapses.-function cleanSentence(text: string): string {- return text- .replace(/!\[[^\]]*\]\([^)]*\)/g, " ") // images- .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") // links -> link text- .replace(/https?:\/\/\S+/g, " ") // bare URLs- .replace(/^\s*\d+\s*/, "") // leading code-block line number- .replace(/[#*`>_~|\\]+/g, " ") // markdown marks- .replace(/\s+/g, " ")- .trim();+interface HighlightResult {+ pruned_markdown?: string;}-// Drop markdown/code artifacts (line numbers, "=====", "//", "[ 01 / 06 ]")-// that the semantic model sometimes scores highly. Real sentences have prose.-function isContentful(text: string): boolean {- const letters = (text.match(/[a-zA-Z]/g) ?? []).length;- return letters >= 15;+interface BatchHighlightResponse {+ results?: HighlightResult[];}-interface HighlightModelResponse {- kept_sentences?: string[];- sentence_probabilities?: number[];+// The service returns the selected spans joined with "\n" (often with a trailing+// newline). Trim to a clean snippet; null when nothing was selected.+function prunedToSnippet(result: HighlightResult | undefined): string | null {+ const pruned = result?.pruned_markdown;+ if (typeof pruned !== "string") return null;+ const trimmed = pruned.trim();+ return trimmed === "" ? null : trimmed;}/**- * Split markdown into chunks under the model's token budget. Splits on blank- * lines (paragraph boundaries) to avoid cutting mid-sentence; oversized- * paragraphs are hard-split as a last resort. Capped at MAX_CHUNKS.+ * Generate query-relevant highlights for many pages in a single batch call to+ * the query-highlights model. Each page's markdown is sent as-is; the service+ * splits it into the line-like spans used in training, scores them against the+ * query, and returns the selected spans joined in page order+ * (`pruned_markdown`). Returns an array aligned to `items`: the highlight+ * snippet for each page, or null when nothing clears the threshold (or the whole+ * batch fails β highlights are best-effort, callers keep the provider snippet).*/-function chunkMarkdown(markdown: string): string[] {- const chunks: string[] = [];- let current = "";-- const flush = () => {- if (current.trim() !== "") chunks.push(current);- current = "";- };-- for (const para of markdown.split(/\n{2,}/)) {- if (chunks.length >= MAX_CHUNKS) break;-- if (para.length > CHUNK_CHARS) {- flush();- for (- let i = 0;- i < para.length && chunks.length < MAX_CHUNKS;- i += CHUNK_CHARS- ) {- chunks.push(para.slice(i, i + CHUNK_CHARS));- }- continue;- }-- if (current.length + para.length + 2 > CHUNK_CHARS) {- flush();- }- current += (current === "" ? "" : "\n\n") + para;- }-- if (chunks.length < MAX_CHUNKS) flush();- return chunks.slice(0, MAX_CHUNKS);-}+export async function generateHighlightsBatch(+ items: HighlightItem[],+ opts: { logger: Logger },+): Promise<(string | null)[]> {+ if (items.length === 0) return [];-async function scoreChunk(- question: string,- context: string,- logger: Logger,-): Promise<{ text: string; score: number }[]> {const controller = new AbortController();const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);+ const start = Date.now();try {- const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/v1/highlight`, {+ const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/batch_highlight`, {method: "POST",- headers: { "Content-Type": "application/json" },+ headers: {+ "Content-Type": "application/json",+ Authorization: `Bearer ${config.HIGHLIGHT_MODEL_TOKEN}`,+ },body: JSON.stringify({- question,- context,- // threshold 0 => every sentence is returned with its score, so we can- // pool scores across chunks and select globally.- threshold: 0.0,- language: "auto",- return_sentence_metrics: true,+ requests: items.map(item => ({+ query: item.query,+ markdown: item.markdown,+ threshold: HIGHLIGHT_THRESHOLD,+ max_highlight_chars: MAX_HIGHLIGHT_CHARS,+ })),}),signal: controller.signal,});@@ -120,84 +83,24 @@ async function scoreChunk();}- const data = (await res.json()) as HighlightModelResponse;- const sentences = data.kept_sentences ?? [];- const probs = data.sentence_probabilities ?? [];- const n = Math.min(sentences.length, probs.length);- const out: { text: string; score: number }[] = [];- for (let i = 0; i < n; i++) {- out.push({ text: sentences[i], score: probs[i] });- }- return out;- } finally {- clearTimeout(timer);- }-}--/**- * Generate query-relevant highlights from page markdown using the semantic- * highlight model. Returns the assembled highlight string (best sentences in- * document order), or null if nothing clears the threshold / all chunks fail.- */-export async function generateSemanticHighlights(- markdown: string,- query: string,- opts: { logger: Logger },-): Promise<string | null> {- const chunks = chunkMarkdown(markdown);- if (chunks.length === 0) return null;+ const data = (await res.json()) as BatchHighlightResponse;+ const results = data.results ?? [];+ const snippets = items.map((_, i) => prunedToSnippet(results[i]));- const start = Date.now();- const perChunk = await Promise.all(- chunks.map(async (chunk, idx) => {- try {- return await scoreChunk(query, chunk, opts.logger);- } catch (error) {- opts.logger.warn("highlight model chunk failed", {- error: error instanceof Error ? error.message : String(error),- chunkIdx: idx,- });- return [];- }- }),- );+ opts.logger.info("query highlights batch generated", {+ pages: items.length,+ withHighlights: snippets.filter(Boolean).length,+ elapsedMs: Date.now() - start,+ });- // Pool every scored sentence, preserving document order (Promise.all keeps- // chunk order, and sentences within a chunk are already in order).- const all: ScoredSentence[] = [];- let order = 0;- for (const chunkSentences of perChunk) {- for (const s of chunkSentences) {- const idx = order++;- const cleaned = cleanSentence(s.text);- if (isContentful(cleaned)) {- all.push({ text: cleaned, score: s.score, order: idx });- }- }+ return snippets;+ } catch (error) {+ opts.logger.warn("query highlights batch failed", {+ error: error instanceof Error ? error.message : String(error),+ pages: items.length,+ });+ return items.map(() => null);+ } finally {+ clearTimeout(timer);}- if (all.length === 0) return null;-- // Pick the best: keep sentences at/above the threshold, cap to the top- // MAX_SELECTED by score, then restore document order for readability.- let selected = all.filter(s => s.score >= SELECT_THRESHOLD);- if (selected.length === 0) return null;- selected.sort((a, b) => b.score - a.score);- selected = selected.slice(0, MAX_SELECTED);- selected.sort((a, b) => a.order - b.order);-- const text = selected- .map(s => s.text.trim())- .join(" ")- .replace(/\s+/g, " ")- .trim();-- opts.logger.info("semantic highlights generated", {- chunks: chunks.length,- scoredSentences: all.length,- selected: selected.length,- topScore: all.reduce((m, s) => Math.max(m, s.score), 0),- elapsedMs: Date.now() - start,- });-- return text === "" ? null : text;}diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.tsindex 4bb4f44c3..9a6eb0a9f 100644--- a/apps/api/src/search/highlights.ts+++ b/apps/api/src/search/highlights.ts@@ -10,7 +10,7 @@ import { indexGetRecent5 } from "../db/rpc";import { parseMarkdown } from "../lib/html-to-markdown";import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";import type { ScrapeOptions } from "../controllers/v2/types";-import { generateSemanticHighlights } from "./highlight-model";+import { generateHighlightsBatch } from "./highlight-model";import { config } from "../config";// How far back into the index we're willing to reach for highlight source text.@@ -19,11 +19,15 @@ const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days/*** Whether the deployment has every dependency the highlights beta needs: the* index DB (to find cached content), the GCS index bucket (to fetch it), and the- * highlight model endpoint (to score it). Missing any => silently skip.+ * highlight model service URL + token (to score it). Missing any => silently+ * skip.*/export function highlightsEnvReady(): boolean {return (- useIndex && !!config.GCS_INDEX_BUCKET_NAME && !!config.HIGHLIGHT_MODEL_URL+ useIndex &&+ !!config.GCS_INDEX_BUCKET_NAME &&+ !!config.HIGHLIGHT_MODEL_URL &&+ !!config.HIGHLIGHT_MODEL_TOKEN);}@@ -115,89 +119,78 @@ async function getIndexedMarkdownForURL(}}-async function highlightOne(- url: string,- query: string,- logger: Logger,- apply: (highlights: string) => void,- counters: { indexHits: number; replaced: number },-): Promise<void> {- const markdown = await getIndexedMarkdownForURL(url, logger);- if (!markdown) {- return;- }- counters.indexHits++;-- const highlights = await generateSemanticHighlights(markdown, query, {- logger,- });- if (highlights && highlights.trim() !== "") {- apply(highlights);- counters.replaced++;- }-}-/**- * For each search result, in parallel: look up the URL in our index (last 30- * days), and if present, replace the provider snippet with query-relevant- * highlights generated from the indexed content. Mutates `response` in place.- * Results not in the index keep their original snippet.+ * For each search result: look up the URL in our index (last 30 days), and if+ * present, replace the provider snippet with query-relevant highlights generated+ * from the indexed content. Index lookups run in parallel, then every hit is+ * scored in a single batch call to the query-highlights model. Mutates+ * `response` in place. Results not in the index keep their original snippet.*/export async function applySearchHighlights(response: SearchV2Response,query: string,logger: Logger,): Promise<{ attempted: number; indexHits: number; replaced: number }> {- const counters = { indexHits: 0, replaced: 0 };- const tasks: Promise<void>[] = [];const start = Date.now();- // Web results carry the snippet in `description` β replace it in place.+ // Collect every result we could highlight, each with a setter for its snippet+ // field: web results carry it in `description`, news results in `snippet`.+ const targets: { url: string; apply: (h: string) => void }[] = [];for (const result of response.web ?? []) {if (!result.url) continue;- tasks.push(- highlightOne(- result.url,- query,- logger,- h => {- result.description = h;- },- counters,- ),- );+ targets.push({+ url: result.url,+ apply: h => {+ result.description = h;+ },+ });}-- // News results carry the snippet in `snippet` β replace it in place.for (const result of response.news ?? []) {if (!result.url) continue;- const url = result.url;- tasks.push(- highlightOne(- url,- query,- logger,- h => {- result.snippet = h;- },- counters,- ),- );+ targets.push({+ url: result.url,+ apply: h => {+ result.snippet = h;+ },+ });}- const attempted = tasks.length;- await Promise.all(tasks);+ const attempted = targets.length;+ if (attempted === 0) {+ return { attempted, indexHits: 0, replaced: 0 };+ }++ // Look up indexed markdown for every URL in parallel, keeping only the hits.+ const markdowns = await Promise.all(+ targets.map(t => getIndexedMarkdownForURL(t.url, logger)),+ );+ const hits: { apply: (h: string) => void; markdown: string }[] = [];+ markdowns.forEach((markdown, i) => {+ if (markdown) hits.push({ apply: targets[i].apply, markdown });+ });+ const indexHits = hits.length;++ // Score every hit in one batch call, then apply the highlights that came back.+ let replaced = 0;+ if (indexHits > 0) {+ const snippets = await generateHighlightsBatch(+ hits.map(h => ({ query, markdown: h.markdown })),+ { logger },+ );+ snippets.forEach((snippet, i) => {+ if (snippet && snippet.trim() !== "") {+ hits[i].apply(snippet);+ replaced++;+ }+ });+ }logger.info("Search highlights applied", {attempted,- indexHits: counters.indexHits,- replaced: counters.replaced,+ indexHits,+ replaced,timeTakenMs: Date.now() - start,});- return {- attempted,- indexHits: counters.indexHits,- replaced: counters.replaced,- };+ return { attempted, indexHits, replaced };}