Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(search): use new /highlight contract for search highlights beta
mogeryd906295ad783
diff --git a/apps/api/src/search/highlight-budget.test.ts b/apps/api/src/search/highlight-budget.test.tsdeleted file mode 100644index 71052b864..000000000--- a/apps/api/src/search/highlight-budget.test.ts+++ /dev/null@@ -1,93 +0,0 @@-import { selectHighlightIndices } from "./highlight-budget";--describe("selectHighlightIndices", () => {- it("returns [] when there are no scored spans", () => {- expect(selectHighlightIndices([10, 10], [])).toEqual([]);- });-- it("ignores out-of-range / invalid span indices", () => {- const out = selectHighlightIndices(- [10, 10],- [- { index: 5, score: 0.9 },- { index: -1, score: 0.8 },- ],- );- expect(out).toEqual([]);- });-- it("expands a selected line with its ±1 neighbors", () => {- const out = selectHighlightIndices(- [10, 10, 10, 10, 10],- [{ index: 2, score: 0.9 }],- );- expect(out).toEqual([1, 2, 3]);- });-- it("does not add neighbors that exceed the neighbor budget", () => {- // Neighbors are 100 chars each; neighbor budget = floor(100 * 0.35) = 35.- const out = selectHighlightIndices(- [100, 100, 10, 100, 100],- [{ index: 2, score: 0.9 }],- { maxChars: 100, neighborBudgetFraction: 0.35 },- );- expect(out).toEqual([2]);- });-- it("gives neighbor budget to the higher-scoring line first", () => {- // Budget allows exactly one 50-char neighbor. index 4 (0.9) should claim it- // (its neighbor 3), leaving index 0 (0.4) without one.- const out = selectHighlightIndices(- [50, 50, 50, 50, 50],- [- { index: 0, score: 0.4 },- { index: 4, score: 0.9 },- ],- { maxChars: 1000, neighborBudgetFraction: 0.05 },- );- expect(out).toEqual([0, 3, 4]);- });-- it("merges adjacent selections (incl. a bridging neighbor) into one block", () => {- const out = selectHighlightIndices(- [10, 10, 10, 10],- [- { index: 0, score: 0.3 },- { index: 2, score: 0.9 },- ],- );- expect(out).toEqual([0, 1, 2, 3]);- });-- it("emits the highest-scoring block first and drops blocks that exceed budget", () => {- // Two non-adjacent blocks of 100 chars; budget fits only one.- const out = selectHighlightIndices(- [100, 100, 100, 100, 100, 100, 100],- [- { index: 0, score: 0.5 },- { index: 6, score: 0.9 },- ],- { maxChars: 150, neighborBudgetFraction: 0 },- );- expect(out).toEqual([6]);- });-- it("returns chosen blocks in page order, not score order", () => {- const out = selectHighlightIndices(- [100, 100, 100, 100, 100, 100, 100],- [- { index: 0, score: 0.5 },- { index: 6, score: 0.9 },- ],- { maxChars: 500, neighborBudgetFraction: 0 },- );- expect(out).toEqual([0, 6]);- });-- it("always keeps the top block even if it alone exceeds the budget", () => {- const out = selectHighlightIndices([1000], [{ index: 0, score: 0.5 }], {- maxChars: 800,- });- expect(out).toEqual([0]);- });-});diff --git a/apps/api/src/search/highlight-budget.ts b/apps/api/src/search/highlight-budget.tsdeleted file mode 100644index b23b66398..000000000--- a/apps/api/src/search/highlight-budget.ts+++ /dev/null@@ -1,96 +0,0 @@-import type { ScoredSpan } from "./highlight-model";--// Turn the model's scored spans into the final set of line indices to emit,-// within a character budget. Two policies, applied in order:-//-// 1. Neighbor policy: each selected line may pull in its ±1 adjacent lines for-// context, but neighbors collectively get at most NEIGHBOR_BUDGET_FRACTION-// of the budget — so context never crowds out actual answer lines.-// 2. Group budget: merge adjacent selected lines into contiguous blocks, rank-// blocks by their best span score, and emit the best blocks (in page order)-// until the budget is reached. Budgeting by block rather than raw line keeps-// a coherent passage together instead of letting many tiny unrelated lines-// each eat into the budget.--// Total character budget for the assembled snippet (approximate — measured on-// raw line text, not the markdown assembleAnswer adds back).-const MAX_HIGHLIGHT_CHARS = 800;-// Neighbor (context) lines may use at most this fraction of the budget.-const NEIGHBOR_BUDGET_FRACTION = 0.35;--interface Block {- indices: number[];- chars: number;- score: number; // best core-span score in the block-}--/**- * Choose which line indices to emit. `lineLengths[i]` is the character length of- * line i (sentences[i].text.length); `scored` are the model's selected spans.- * Returns indices in ascending (page) order, ready for assembleAnswer.- */-export function selectHighlightIndices(- lineLengths: number[],- scored: ScoredSpan[],- opts: { maxChars?: number; neighborBudgetFraction?: number } = {},-): number[] {- const maxChars = opts.maxChars ?? MAX_HIGHLIGHT_CHARS;- const neighborFraction =- opts.neighborBudgetFraction ?? NEIGHBOR_BUDGET_FRACTION;- const n = lineLengths.length;-- // Core selected lines (valid, in range). Keep the best score per index.- const coreScore = new Map<number, number>();- for (const s of scored) {- if (!Number.isInteger(s.index) || s.index < 0 || s.index >= n) continue;- const prev = coreScore.get(s.index);- if (prev === undefined || s.score > prev) coreScore.set(s.index, s.score);- }- if (coreScore.size === 0) return [];-- // Process core lines best-score first so higher-value lines claim neighbor- // budget before lower-value ones.- const coreByScore = [...coreScore.entries()].sort((a, b) => b[1] - a[1]);-- // Neighbor expansion (±1 line), capped at a fraction of the total budget.- const neighborBudget = Math.floor(maxChars * neighborFraction);- const selected = new Set<number>(coreScore.keys());- let neighborChars = 0;- for (const [idx] of coreByScore) {- for (const nb of [idx - 1, idx + 1]) {- if (nb < 0 || nb >= n) continue;- if (selected.has(nb)) continue;- if (neighborChars + lineLengths[nb] > neighborBudget) continue;- selected.add(nb);- neighborChars += lineLengths[nb];- }- }-- // Merge adjacent selected indices into contiguous blocks.- const sorted = [...selected].sort((a, b) => a - b);- const blocks: Block[] = [];- for (const idx of sorted) {- const last = blocks[blocks.length - 1];- const score = coreScore.get(idx) ?? -Infinity;- if (last && idx === last.indices[last.indices.length - 1] + 1) {- last.indices.push(idx);- last.chars += lineLengths[idx];- last.score = Math.max(last.score, score);- } else {- blocks.push({ indices: [idx], chars: lineLengths[idx], score });- }- }-- // Rank blocks by best score and emit until the budget is reached. Always keep- // the top block so we return something even if it alone exceeds the budget.- blocks.sort((a, b) => b.score - a.score);- const chosen: Block[] = [];- let total = 0;- for (const b of blocks) {- if (chosen.length > 0 && total + b.chars > maxChars) continue;- chosen.push(b);- total += b.chars;- }-- return chosen.flatMap(b => b.indices).sort((a, b) => a - b);-}diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.tsindex 0a0020e4e..23cc19ba4 100644--- a/apps/api/src/search/highlight-model.test.ts+++ b/apps/api/src/search/highlight-model.test.ts@@ -5,7 +5,7 @@ vi.mock("../config", () => ({},}));-import { generateHighlightsBatch } from "./highlight-model";+import { generateHighlights } from "./highlight-model";const logger = {info: vi.fn(),@@ -30,129 +30,73 @@ afterEach(() => {vi.clearAllMocks();});-describe("generateHighlightsBatch", () => {- it("returns [] without calling the service for an empty batch", async () => {- const fetchMock = mockFetchOnce({ results: [] });- const out = await generateHighlightsBatch([], { logger });- expect(out).toEqual([]);- expect(fetchMock).not.toHaveBeenCalled();- });-- it("posts each page's spans as `lines` to /batch_highlight with the bearer token", async () => {- const fetchMock = mockFetchOnce({- results: [{ highlights: [] }, { highlights: [] }],- });+describe("generateHighlights", () => {+ it("posts the query and full markdown to /highlight with the bearer token", async () => {+ const fetchMock = mockFetchOnce({ highlights: [], markdown: "" });- await generateHighlightsBatch(- [- { query: "q1", lines: ["a one", "b one"] },- { query: "q2", lines: ["a two"] },- ],- { logger },- );+ await generateHighlights("q1", "# Page\n\nbody text", { logger });expect(fetchMock).toHaveBeenCalledTimes(1);const [url, init] = fetchMock.mock.calls[0];- expect(url).toBe("https://highlight.test/batch_highlight");+ expect(url).toBe("https://highlight.test/highlight");expect(init.method).toBe("POST");expect(init.headers.Authorization).toBe("Bearer secret-token");const sent = JSON.parse(init.body);- expect(sent.requests).toHaveLength(2);- expect(sent.requests[0]).toMatchObject({- query: "q1",- lines: ["a one", "b one"],- });- // Threshold + top_k are sent; char budgeting is done client-side (group-aware),- // so max_highlight_chars is intentionally NOT sent.- expect(typeof sent.requests[0].threshold).toBe("number");- expect(typeof sent.requests[0].top_k).toBe("number");- expect(sent.requests[0].max_highlight_chars).toBeUndefined();+ expect(sent).toEqual({ query: "q1", markdown: "# Page\n\nbody text" });});- it("returns the scored spans per page, aligned by request order", async () => {+ it("returns the highlights and reassembled markdown from the service", async () => {mockFetchOnce({- results: [- {- highlights: [- { index: 0, score: 0.9 },- { index: 2, score: 0.5 },- ],- },- { highlights: [{ index: 1, score: 0.7 }] },+ highlights: [+ { block_index: 0, score: 0.9 },+ { block_index: 2, score: 0.5 },],+ markdown: "reassembled answer",});- const out = await generateHighlightsBatch(- [- { query: "q1", lines: ["l0", "l1", "l2"] },- { query: "q2", lines: ["l0", "l1"] },- ],- { logger },- );+ const out = await generateHighlights("q", "some markdown", { logger });- expect(out).toEqual([- [- { index: 0, score: 0.9 },- { index: 2, score: 0.5 },+ expect(out).toEqual({+ highlights: [+ { block_index: 0, score: 0.9 },+ { block_index: 2, score: 0.5 },],- [{ index: 1, score: 0.7 }],- ]);+ markdown: "reassembled answer",+ });});- it("returns an empty array for a page with no selected spans", async () => {- mockFetchOnce({ results: [{ highlights: [] }, {}] });+ it("debug-logs the highlights array", async () => {+ mockFetchOnce({+ highlights: [{ block_index: 1, score: 0.8 }],+ markdown: "x",+ });- const out = await generateHighlightsBatch(- [- { query: "q", lines: ["a"] },- { query: "q", lines: ["b"] },- ],- { logger },- );+ await generateHighlights("q", "md", { logger });- expect(out).toEqual([[], []]);+ expect(logger.debug).toHaveBeenCalledWith(+ "query highlights",+ expect.objectContaining({+ canonicalLog: "search/highlights",+ highlights: [{ block_index: 1, score: 0.8 }],+ }),+ );});- it("ignores malformed (non-integer index / missing score) spans", async () => {- mockFetchOnce({- results: [- {- highlights: [- { index: 1, score: 0.9 },- { index: 4 }, // missing score- { index: 2.5, score: 0.7 }, // non-integer index- { index: 3, score: 0.6 },- ],- },- ],- });+ it("defaults missing fields to empty highlights and markdown", async () => {+ mockFetchOnce({});- const out = await generateHighlightsBatch(- [{ query: "q", lines: ["a", "b", "c", "d", "e"] }],- { logger },- );+ const out = await generateHighlights("q", "md", { logger });- expect(out).toEqual([- [- { index: 1, score: 0.9 },- { index: 3, score: 0.6 },- ],- ]);+ expect(out).toEqual({ highlights: [], markdown: "" });});- it("falls back to nulls (one per item) when the service errors", async () => {+ it("returns null when the service errors", async () => {mockFetchOnce({ error: "boom" }, false, 500);- const out = await generateHighlightsBatch(- [- { query: "q", lines: ["a"] },- { query: "q", lines: ["b"] },- ],- { logger },- );+ const out = await generateHighlights("q", "md", { logger });- expect(out).toEqual([null, null]);+ expect(out).toBeNull();expect(logger.warn).toHaveBeenCalled();});});diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.tsindex c785ae3cf..427c9543a 100644--- a/apps/api/src/search/highlight-model.ts+++ b/apps/api/src/search/highlight-model.ts@@ -1,101 +1,59 @@import type { Logger } from "winston";import { config } from "../config";-// Query Highlights model service: a query-finetuned span scorer (no LLM). It was-// trained on the line-like spans `parseMarkdownToSentences` produces, so callers-// pass those spans as explicit `lines`, the service scores each against the-// query, and returns the selected spans as `highlights[]` with their `index`-// back into the input `lines` array (and a `score`). Callers run those scored-// spans through the neighbor/group budgeter, then `assembleAnswer` to rebuild-// structure (tables/code). We push every page through one `/batch_highlight`-// call (resident GPU model). Endpoint + token are config-gated+// Query Highlights model service: given a page's full markdown and a query, the+// service returns the query-relevant highlights plus those highlights+// reassembled into a single markdown document (in document order). We push each+// page through one `/highlight` call. Endpoint + token are config-gated// (HIGHLIGHT_MODEL_URL / HIGHLIGHT_MODEL_TOKEN); callers must confirm they're// set via highlightsEnvReady() before invoking.-// Experimental score cutoff for keeping a span, tuned against the line-span-// format the model was trained on.-const HIGHLIGHT_THRESHOLD = 0.08;-// Cap on how many spans to keep per page after thresholding (highest-scoring).-// We do char budgeting ourselves (group-aware, in selectHighlightIndices), so we-// don't send max_highlight_chars — the service would budget by raw lines.-const HIGHLIGHT_TOP_K = 12;const REQUEST_TIMEOUT_MS = 30000;-// One scored candidate span: `index` into the page's `lines`, `score` from the-// model. Consumed by the neighbor/group budgeter (selectHighlightIndices).-export interface ScoredSpan {- index: number;- score: number;-}--interface HighlightItem {- query: string;- // Candidate spans for the page, in document order — from- // parseMarkdownToSentences(markdown).map(s => s.text).- lines: string[];-}--interface HighlightSpan {- index?: number;+// One highlight entry as returned by the service. Field semantics are+// intentionally left undocumented here.+interface Highlight {+ block_index?: number;+ kind?: string;+ via?: string;score?: number;+ span_md?: string;}-interface HighlightResult {- highlights?: HighlightSpan[];+interface HighlightResponse {+ highlights?: Highlight[];+ markdown?: string;}-interface BatchHighlightResponse {- results?: HighlightResult[];-}--// Pull the scored spans out of one page's result, keeping only well-formed ones-// (integer index, numeric score). The service already applied threshold + top_k.-function scoredSpans(result: HighlightResult | undefined): ScoredSpan[] {- const highlights = result?.highlights ?? [];- const out: ScoredSpan[] = [];- for (const h of highlights) {- if (- typeof h.index === "number" &&- Number.isInteger(h.index) &&- typeof h.score === "number"- ) {- out.push({ index: h.index, score: h.score });- }- }- return out;+// Result of one highlight call: the service's highlight entries plus the+// reassembled markdown document.+interface HighlightResult {+ highlights: Highlight[];+ markdown: string;}/**- * Score many pages' candidate spans in a single batch call to the- * query-highlights model. Returns an array aligned to `items`: for each page,- * the scored spans (index into that page's `lines`, plus score) the model- * selected, or null when the whole batch fails (so callers can keep the provider- * snippet). A page with no spans clearing the threshold yields an empty array.+ * Generate query highlights for one page by sending its full markdown and the+ * query to the highlight model service. Returns the service's highlight entries+ * plus the reassembled markdown, or null when the call fails (so callers can+ * keep the provider snippet).*/-export async function generateHighlightsBatch(- items: HighlightItem[],+export async function generateHighlights(+ query: string,+ markdown: string,opts: { logger: Logger },-): Promise<(ScoredSpan[] | null)[]> {- if (items.length === 0) return [];-+): Promise<HighlightResult | null> {const controller = new AbortController();const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);const start = Date.now();try {- const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/batch_highlight`, {+ const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/highlight`, {method: "POST",headers: {"Content-Type": "application/json",Authorization: `Bearer ${config.HIGHLIGHT_MODEL_TOKEN}`,},- body: JSON.stringify({- requests: items.map(item => ({- query: item.query,- lines: item.lines,- threshold: HIGHLIGHT_THRESHOLD,- top_k: HIGHLIGHT_TOP_K,- })),- }),+ body: JSON.stringify({ query, markdown }),signal: controller.signal,});@@ -106,23 +64,23 @@ export async function generateHighlightsBatch();}- const data = (await res.json()) as BatchHighlightResponse;- const results = data.results ?? [];- const spans = items.map((_, i) => scoredSpans(results[i]));+ const data = (await res.json()) as HighlightResponse;+ const highlights = data.highlights ?? [];- opts.logger.info("query highlights batch generated", {- pages: items.length,- withHighlights: spans.filter(x => x.length > 0).length,+ // Canonical log of the highlights array, debug-level only.+ opts.logger.debug("query highlights", {+ canonicalLog: "search/highlights",+ highlights,elapsedMs: Date.now() - start,});- return spans;+ return { highlights, markdown: data.markdown ?? "" };} catch (error) {- opts.logger.warn("query highlights batch failed", {+ opts.logger.warn("query highlights failed", {+ canonicalLog: "search/highlights",error: error instanceof Error ? error.message : String(error),- pages: items.length,});- return items.map(() => null);+ return null;} finally {clearTimeout(timer);}diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.tsindex dcd6cd69a..60291a196 100644--- a/apps/api/src/search/highlights.ts+++ b/apps/api/src/search/highlights.ts@@ -10,12 +10,7 @@ import { indexGetRecent5 } from "../db/rpc";import { parseMarkdown } from "../lib/html-to-markdown";import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";import type { ScrapeOptions } from "../controllers/v2/types";-import { generateHighlightsBatch } from "./highlight-model";-import { selectHighlightIndices } from "./highlight-budget";-import {- parseMarkdownToSentences,- assembleAnswer,-} from "../lib/highlight-spans";+import { generateHighlights } from "./highlight-model";import { config } from "../config";// How far back into the index we're willing to reach for highlight source text.@@ -127,12 +122,10 @@ async function getIndexedMarkdownForURL(/*** For each search result: look up the URL in our index (last 30 days), and if* present, replace the provider snippet with query-relevant highlights generated- * from the indexed content. Index lookups run in parallel; each hit's markdown- * is split into the same candidate spans the highlight model was trained on- * (parseMarkdownToSentences), every hit is scored in a single batch call, and- * the selected span indices are reassembled with assembleAnswer (structure-aware- * — rebuilds tables/code). Mutates `response` in place. Results not in the index- * keep their original snippet.+ * from the indexed content. Index lookups run in parallel; each hit's full+ * markdown is sent to the highlight model service, which returns the selected+ * highlights reassembled into a single markdown document. Mutates `response` in+ * place. Results not in the index keep their original snippet.*/export async function applySearchHighlights(response: SearchV2Response,@@ -168,39 +161,31 @@ export async function applySearchHighlights(return { attempted, indexHits: 0, replaced: 0 };}- // Look up indexed markdown for every URL in parallel, then split each hit into- // candidate spans. Keep the parsed sentences around so we can reassemble the- // model's selected indices back into structured markdown.+ // Look up indexed markdown for every URL in parallel, keeping the markdown for+ // each hit so we can send it to the highlight model service.const markdowns = await Promise.all(targets.map(t => getIndexedMarkdownForURL(t.url, logger)),);const hits: {apply: (h: string) => void;- sentences: ReturnType<typeof parseMarkdownToSentences>;+ markdown: string;}[] = [];markdowns.forEach((markdown, i) => {if (!markdown) return;- const sentences = parseMarkdownToSentences(markdown);- if (sentences.length === 0) return;- hits.push({ apply: targets[i].apply, sentences });+ hits.push({ apply: targets[i].apply, markdown });});const indexHits = hits.length;- // Score every hit in one batch call (candidate spans as explicit `lines`),- // then for each page: budget the scored spans (neighbor + group policy) and- // reassemble the chosen indices into a snippet.+ // Send each hit's full markdown to the highlight model service in parallel and+ // use the reassembled markdown it returns as the snippet.let replaced = 0;if (indexHits > 0) {- const perPageSpans = await generateHighlightsBatch(- hits.map(h => ({ query, lines: h.sentences.map(s => s.text) })),- { logger },+ const results = await Promise.all(+ hits.map(h => generateHighlights(query, h.markdown, { logger })),);- perPageSpans.forEach((spans, i) => {- if (!spans || spans.length === 0) return;- const lineLengths = hits[i].sentences.map(s => s.text.length);- const indices = selectHighlightIndices(lineLengths, spans);- if (indices.length === 0) return;- const snippet = assembleAnswer(hits[i].sentences, indices);+ results.forEach((result, i) => {+ if (!result) return;+ const snippet = result.markdown;if (snippet.trim() !== "") {hits[i].apply(snippet);replaced++;