Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(search): use new /highlight contract for search highlights beta

mogeryd906295ad783
diff --git a/apps/api/src/search/highlight-budget.test.ts b/apps/api/src/search/highlight-budget.test.ts
deleted file mode 100644
index 71052b864..000000000
--- a/apps/api/src/search/highlight-budget.test.ts
+++ /dev/null
@@ -1,93 +0,0 @@
-import { selectHighlightIndices } from "./highlight-budget";
-
-describe("selectHighlightIndices", () => {
- it("returns [] when there are no scored spans", () => {
- expect(selectHighlightIndices([10, 10], [])).toEqual([]);
- });
-
- it("ignores out-of-range / invalid span indices", () => {
- const out = selectHighlightIndices(
- [10, 10],
- [
- { index: 5, score: 0.9 },
- { index: -1, score: 0.8 },
- ],
- );
- expect(out).toEqual([]);
- });
-
- it("expands a selected line with its ±1 neighbors", () => {
- const out = selectHighlightIndices(
- [10, 10, 10, 10, 10],
- [{ index: 2, score: 0.9 }],
- );
- expect(out).toEqual([1, 2, 3]);
- });
-
- it("does not add neighbors that exceed the neighbor budget", () => {
- // Neighbors are 100 chars each; neighbor budget = floor(100 * 0.35) = 35.
- const out = selectHighlightIndices(
- [100, 100, 10, 100, 100],
- [{ index: 2, score: 0.9 }],
- { maxChars: 100, neighborBudgetFraction: 0.35 },
- );
- expect(out).toEqual([2]);
- });
-
- it("gives neighbor budget to the higher-scoring line first", () => {
- // Budget allows exactly one 50-char neighbor. index 4 (0.9) should claim it
- // (its neighbor 3), leaving index 0 (0.4) without one.
- const out = selectHighlightIndices(
- [50, 50, 50, 50, 50],
- [
- { index: 0, score: 0.4 },
- { index: 4, score: 0.9 },
- ],
- { maxChars: 1000, neighborBudgetFraction: 0.05 },
- );
- expect(out).toEqual([0, 3, 4]);
- });
-
- it("merges adjacent selections (incl. a bridging neighbor) into one block", () => {
- const out = selectHighlightIndices(
- [10, 10, 10, 10],
- [
- { index: 0, score: 0.3 },
- { index: 2, score: 0.9 },
- ],
- );
- expect(out).toEqual([0, 1, 2, 3]);
- });
-
- it("emits the highest-scoring block first and drops blocks that exceed budget", () => {
- // Two non-adjacent blocks of 100 chars; budget fits only one.
- const out = selectHighlightIndices(
- [100, 100, 100, 100, 100, 100, 100],
- [
- { index: 0, score: 0.5 },
- { index: 6, score: 0.9 },
- ],
- { maxChars: 150, neighborBudgetFraction: 0 },
- );
- expect(out).toEqual([6]);
- });
-
- it("returns chosen blocks in page order, not score order", () => {
- const out = selectHighlightIndices(
- [100, 100, 100, 100, 100, 100, 100],
- [
- { index: 0, score: 0.5 },
- { index: 6, score: 0.9 },
- ],
- { maxChars: 500, neighborBudgetFraction: 0 },
- );
- expect(out).toEqual([0, 6]);
- });
-
- it("always keeps the top block even if it alone exceeds the budget", () => {
- const out = selectHighlightIndices([1000], [{ index: 0, score: 0.5 }], {
- maxChars: 800,
- });
- expect(out).toEqual([0]);
- });
-});
diff --git a/apps/api/src/search/highlight-budget.ts b/apps/api/src/search/highlight-budget.ts
deleted file mode 100644
index b23b66398..000000000
--- a/apps/api/src/search/highlight-budget.ts
+++ /dev/null
@@ -1,96 +0,0 @@
-import type { ScoredSpan } from "./highlight-model";
-
-// Turn the model's scored spans into the final set of line indices to emit,
-// within a character budget. Two policies, applied in order:
-//
-// 1. Neighbor policy: each selected line may pull in its ±1 adjacent lines for
-// context, but neighbors collectively get at most NEIGHBOR_BUDGET_FRACTION
-// of the budget — so context never crowds out actual answer lines.
-// 2. Group budget: merge adjacent selected lines into contiguous blocks, rank
-// blocks by their best span score, and emit the best blocks (in page order)
-// until the budget is reached. Budgeting by block rather than raw line keeps
-// a coherent passage together instead of letting many tiny unrelated lines
-// each eat into the budget.
-
-// Total character budget for the assembled snippet (approximate — measured on
-// raw line text, not the markdown assembleAnswer adds back).
-const MAX_HIGHLIGHT_CHARS = 800;
-// Neighbor (context) lines may use at most this fraction of the budget.
-const NEIGHBOR_BUDGET_FRACTION = 0.35;
-
-interface Block {
- indices: number[];
- chars: number;
- score: number; // best core-span score in the block
-}
-
-/**
- * Choose which line indices to emit. `lineLengths[i]` is the character length of
- * line i (sentences[i].text.length); `scored` are the model's selected spans.
- * Returns indices in ascending (page) order, ready for assembleAnswer.
- */
-export function selectHighlightIndices(
- lineLengths: number[],
- scored: ScoredSpan[],
- opts: { maxChars?: number; neighborBudgetFraction?: number } = {},
-): number[] {
- const maxChars = opts.maxChars ?? MAX_HIGHLIGHT_CHARS;
- const neighborFraction =
- opts.neighborBudgetFraction ?? NEIGHBOR_BUDGET_FRACTION;
- const n = lineLengths.length;
-
- // Core selected lines (valid, in range). Keep the best score per index.
- const coreScore = new Map<number, number>();
- for (const s of scored) {
- if (!Number.isInteger(s.index) || s.index < 0 || s.index >= n) continue;
- const prev = coreScore.get(s.index);
- if (prev === undefined || s.score > prev) coreScore.set(s.index, s.score);
- }
- if (coreScore.size === 0) return [];
-
- // Process core lines best-score first so higher-value lines claim neighbor
- // budget before lower-value ones.
- const coreByScore = [...coreScore.entries()].sort((a, b) => b[1] - a[1]);
-
- // Neighbor expansion (±1 line), capped at a fraction of the total budget.
- const neighborBudget = Math.floor(maxChars * neighborFraction);
- const selected = new Set<number>(coreScore.keys());
- let neighborChars = 0;
- for (const [idx] of coreByScore) {
- for (const nb of [idx - 1, idx + 1]) {
- if (nb < 0 || nb >= n) continue;
- if (selected.has(nb)) continue;
- if (neighborChars + lineLengths[nb] > neighborBudget) continue;
- selected.add(nb);
- neighborChars += lineLengths[nb];
- }
- }
-
- // Merge adjacent selected indices into contiguous blocks.
- const sorted = [...selected].sort((a, b) => a - b);
- const blocks: Block[] = [];
- for (const idx of sorted) {
- const last = blocks[blocks.length - 1];
- const score = coreScore.get(idx) ?? -Infinity;
- if (last && idx === last.indices[last.indices.length - 1] + 1) {
- last.indices.push(idx);
- last.chars += lineLengths[idx];
- last.score = Math.max(last.score, score);
- } else {
- blocks.push({ indices: [idx], chars: lineLengths[idx], score });
- }
- }
-
- // Rank blocks by best score and emit until the budget is reached. Always keep
- // the top block so we return something even if it alone exceeds the budget.
- blocks.sort((a, b) => b.score - a.score);
- const chosen: Block[] = [];
- let total = 0;
- for (const b of blocks) {
- if (chosen.length > 0 && total + b.chars > maxChars) continue;
- chosen.push(b);
- total += b.chars;
- }
-
- return chosen.flatMap(b => b.indices).sort((a, b) => a - b);
-}
diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.ts
index 0a0020e4e..23cc19ba4 100644
--- a/apps/api/src/search/highlight-model.test.ts
+++ b/apps/api/src/search/highlight-model.test.ts
@@ -5,7 +5,7 @@ vi.mock("../config", () => ({
},
}));
-import { generateHighlightsBatch } from "./highlight-model";
+import { generateHighlights } from "./highlight-model";
const logger = {
info: vi.fn(),
@@ -30,129 +30,73 @@ afterEach(() => {
vi.clearAllMocks();
});
-describe("generateHighlightsBatch", () => {
- it("returns [] without calling the service for an empty batch", async () => {
- const fetchMock = mockFetchOnce({ results: [] });
- const out = await generateHighlightsBatch([], { logger });
- expect(out).toEqual([]);
- expect(fetchMock).not.toHaveBeenCalled();
- });
-
- it("posts each page's spans as `lines` to /batch_highlight with the bearer token", async () => {
- const fetchMock = mockFetchOnce({
- results: [{ highlights: [] }, { highlights: [] }],
- });
+describe("generateHighlights", () => {
+ it("posts the query and full markdown to /highlight with the bearer token", async () => {
+ const fetchMock = mockFetchOnce({ highlights: [], markdown: "" });
- await generateHighlightsBatch(
- [
- { query: "q1", lines: ["a one", "b one"] },
- { query: "q2", lines: ["a two"] },
- ],
- { logger },
- );
+ await generateHighlights("q1", "# Page\n\nbody text", { logger });
expect(fetchMock).toHaveBeenCalledTimes(1);
const [url, init] = fetchMock.mock.calls[0];
- expect(url).toBe("https://highlight.test/batch_highlight");
+ expect(url).toBe("https://highlight.test/highlight");
expect(init.method).toBe("POST");
expect(init.headers.Authorization).toBe("Bearer secret-token");
const sent = JSON.parse(init.body);
- expect(sent.requests).toHaveLength(2);
- expect(sent.requests[0]).toMatchObject({
- query: "q1",
- lines: ["a one", "b one"],
- });
- // Threshold + top_k are sent; char budgeting is done client-side (group-aware),
- // so max_highlight_chars is intentionally NOT sent.
- expect(typeof sent.requests[0].threshold).toBe("number");
- expect(typeof sent.requests[0].top_k).toBe("number");
- expect(sent.requests[0].max_highlight_chars).toBeUndefined();
+ expect(sent).toEqual({ query: "q1", markdown: "# Page\n\nbody text" });
});
- it("returns the scored spans per page, aligned by request order", async () => {
+ it("returns the highlights and reassembled markdown from the service", async () => {
mockFetchOnce({
- results: [
- {
- highlights: [
- { index: 0, score: 0.9 },
- { index: 2, score: 0.5 },
- ],
- },
- { highlights: [{ index: 1, score: 0.7 }] },
+ highlights: [
+ { block_index: 0, score: 0.9 },
+ { block_index: 2, score: 0.5 },
],
+ markdown: "reassembled answer",
});
- const out = await generateHighlightsBatch(
- [
- { query: "q1", lines: ["l0", "l1", "l2"] },
- { query: "q2", lines: ["l0", "l1"] },
- ],
- { logger },
- );
+ const out = await generateHighlights("q", "some markdown", { logger });
- expect(out).toEqual([
- [
- { index: 0, score: 0.9 },
- { index: 2, score: 0.5 },
+ expect(out).toEqual({
+ highlights: [
+ { block_index: 0, score: 0.9 },
+ { block_index: 2, score: 0.5 },
],
- [{ index: 1, score: 0.7 }],
- ]);
+ markdown: "reassembled answer",
+ });
});
- it("returns an empty array for a page with no selected spans", async () => {
- mockFetchOnce({ results: [{ highlights: [] }, {}] });
+ it("debug-logs the highlights array", async () => {
+ mockFetchOnce({
+ highlights: [{ block_index: 1, score: 0.8 }],
+ markdown: "x",
+ });
- const out = await generateHighlightsBatch(
- [
- { query: "q", lines: ["a"] },
- { query: "q", lines: ["b"] },
- ],
- { logger },
- );
+ await generateHighlights("q", "md", { logger });
- expect(out).toEqual([[], []]);
+ expect(logger.debug).toHaveBeenCalledWith(
+ "query highlights",
+ expect.objectContaining({
+ canonicalLog: "search/highlights",
+ highlights: [{ block_index: 1, score: 0.8 }],
+ }),
+ );
});
- it("ignores malformed (non-integer index / missing score) spans", async () => {
- mockFetchOnce({
- results: [
- {
- highlights: [
- { index: 1, score: 0.9 },
- { index: 4 }, // missing score
- { index: 2.5, score: 0.7 }, // non-integer index
- { index: 3, score: 0.6 },
- ],
- },
- ],
- });
+ it("defaults missing fields to empty highlights and markdown", async () => {
+ mockFetchOnce({});
- const out = await generateHighlightsBatch(
- [{ query: "q", lines: ["a", "b", "c", "d", "e"] }],
- { logger },
- );
+ const out = await generateHighlights("q", "md", { logger });
- expect(out).toEqual([
- [
- { index: 1, score: 0.9 },
- { index: 3, score: 0.6 },
- ],
- ]);
+ expect(out).toEqual({ highlights: [], markdown: "" });
});
- it("falls back to nulls (one per item) when the service errors", async () => {
+ it("returns null when the service errors", async () => {
mockFetchOnce({ error: "boom" }, false, 500);
- const out = await generateHighlightsBatch(
- [
- { query: "q", lines: ["a"] },
- { query: "q", lines: ["b"] },
- ],
- { logger },
- );
+ const out = await generateHighlights("q", "md", { logger });
- expect(out).toEqual([null, null]);
+ expect(out).toBeNull();
expect(logger.warn).toHaveBeenCalled();
});
});
diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.ts
index c785ae3cf..427c9543a 100644
--- a/apps/api/src/search/highlight-model.ts
+++ b/apps/api/src/search/highlight-model.ts
@@ -1,101 +1,59 @@
import type { Logger } from "winston";
import { config } from "../config";
-// Query Highlights model service: a query-finetuned span scorer (no LLM). It was
-// trained on the line-like spans `parseMarkdownToSentences` produces, so callers
-// pass those spans as explicit `lines`, the service scores each against the
-// query, and returns the selected spans as `highlights[]` with their `index`
-// back into the input `lines` array (and a `score`). Callers run those scored
-// spans through the neighbor/group budgeter, then `assembleAnswer` to rebuild
-// structure (tables/code). We push every page through one `/batch_highlight`
-// call (resident GPU model). Endpoint + token are config-gated
+// Query Highlights model service: given a page's full markdown and a query, the
+// service returns the query-relevant highlights plus those highlights
+// reassembled into a single markdown document (in document order). We push each
+// page through one `/highlight` call. Endpoint + token are config-gated
// (HIGHLIGHT_MODEL_URL / HIGHLIGHT_MODEL_TOKEN); callers must confirm they're
// set via highlightsEnvReady() before invoking.
-// Experimental score cutoff for keeping a span, tuned against the line-span
-// format the model was trained on.
-const HIGHLIGHT_THRESHOLD = 0.08;
-// Cap on how many spans to keep per page after thresholding (highest-scoring).
-// We do char budgeting ourselves (group-aware, in selectHighlightIndices), so we
-// don't send max_highlight_chars — the service would budget by raw lines.
-const HIGHLIGHT_TOP_K = 12;
const REQUEST_TIMEOUT_MS = 30000;
-// One scored candidate span: `index` into the page's `lines`, `score` from the
-// model. Consumed by the neighbor/group budgeter (selectHighlightIndices).
-export interface ScoredSpan {
- index: number;
- score: number;
-}
-
-interface HighlightItem {
- query: string;
- // Candidate spans for the page, in document order — from
- // parseMarkdownToSentences(markdown).map(s => s.text).
- lines: string[];
-}
-
-interface HighlightSpan {
- index?: number;
+// One highlight entry as returned by the service. Field semantics are
+// intentionally left undocumented here.
+interface Highlight {
+ block_index?: number;
+ kind?: string;
+ via?: string;
score?: number;
+ span_md?: string;
}
-interface HighlightResult {
- highlights?: HighlightSpan[];
+interface HighlightResponse {
+ highlights?: Highlight[];
+ markdown?: string;
}
-interface BatchHighlightResponse {
- results?: HighlightResult[];
-}
-
-// Pull the scored spans out of one page's result, keeping only well-formed ones
-// (integer index, numeric score). The service already applied threshold + top_k.
-function scoredSpans(result: HighlightResult | undefined): ScoredSpan[] {
- const highlights = result?.highlights ?? [];
- const out: ScoredSpan[] = [];
- for (const h of highlights) {
- if (
- typeof h.index === "number" &&
- Number.isInteger(h.index) &&
- typeof h.score === "number"
- ) {
- out.push({ index: h.index, score: h.score });
- }
- }
- return out;
+// Result of one highlight call: the service's highlight entries plus the
+// reassembled markdown document.
+interface HighlightResult {
+ highlights: Highlight[];
+ markdown: string;
}
/**
- * Score many pages' candidate spans in a single batch call to the
- * query-highlights model. Returns an array aligned to `items`: for each page,
- * the scored spans (index into that page's `lines`, plus score) the model
- * selected, or null when the whole batch fails (so callers can keep the provider
- * snippet). A page with no spans clearing the threshold yields an empty array.
+ * Generate query highlights for one page by sending its full markdown and the
+ * query to the highlight model service. Returns the service's highlight entries
+ * plus the reassembled markdown, or null when the call fails (so callers can
+ * keep the provider snippet).
*/
-export async function generateHighlightsBatch(
- items: HighlightItem[],
+export async function generateHighlights(
+ query: string,
+ markdown: string,
opts: { logger: Logger },
-): Promise<(ScoredSpan[] | null)[]> {
- if (items.length === 0) return [];
-
+): Promise<HighlightResult | null> {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
const start = Date.now();
try {
- const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/batch_highlight`, {
+ const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/highlight`, {
method: "POST",
headers: {
"Content-Type": "application/json",
Authorization: `Bearer ${config.HIGHLIGHT_MODEL_TOKEN}`,
},
- body: JSON.stringify({
- requests: items.map(item => ({
- query: item.query,
- lines: item.lines,
- threshold: HIGHLIGHT_THRESHOLD,
- top_k: HIGHLIGHT_TOP_K,
- })),
- }),
+ body: JSON.stringify({ query, markdown }),
signal: controller.signal,
});
@@ -106,23 +64,23 @@ export async function generateHighlightsBatch(
);
}
- const data = (await res.json()) as BatchHighlightResponse;
- const results = data.results ?? [];
- const spans = items.map((_, i) => scoredSpans(results[i]));
+ const data = (await res.json()) as HighlightResponse;
+ const highlights = data.highlights ?? [];
- opts.logger.info("query highlights batch generated", {
- pages: items.length,
- withHighlights: spans.filter(x => x.length > 0).length,
+ // Canonical log of the highlights array, debug-level only.
+ opts.logger.debug("query highlights", {
+ canonicalLog: "search/highlights",
+ highlights,
elapsedMs: Date.now() - start,
});
- return spans;
+ return { highlights, markdown: data.markdown ?? "" };
} catch (error) {
- opts.logger.warn("query highlights batch failed", {
+ opts.logger.warn("query highlights failed", {
+ canonicalLog: "search/highlights",
error: error instanceof Error ? error.message : String(error),
- pages: items.length,
});
- return items.map(() => null);
+ return null;
} finally {
clearTimeout(timer);
}
diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts
index dcd6cd69a..60291a196 100644
--- a/apps/api/src/search/highlights.ts
+++ b/apps/api/src/search/highlights.ts
@@ -10,12 +10,7 @@ import { indexGetRecent5 } from "../db/rpc";
import { parseMarkdown } from "../lib/html-to-markdown";
import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";
import type { ScrapeOptions } from "../controllers/v2/types";
-import { generateHighlightsBatch } from "./highlight-model";
-import { selectHighlightIndices } from "./highlight-budget";
-import {
- parseMarkdownToSentences,
- assembleAnswer,
-} from "../lib/highlight-spans";
+import { generateHighlights } from "./highlight-model";
import { config } from "../config";
// How far back into the index we're willing to reach for highlight source text.
@@ -127,12 +122,10 @@ async function getIndexedMarkdownForURL(
/**
* For each search result: look up the URL in our index (last 30 days), and if
* present, replace the provider snippet with query-relevant highlights generated
- * from the indexed content. Index lookups run in parallel; each hit's markdown
- * is split into the same candidate spans the highlight model was trained on
- * (parseMarkdownToSentences), every hit is scored in a single batch call, and
- * the selected span indices are reassembled with assembleAnswer (structure-aware
- * — rebuilds tables/code). Mutates `response` in place. Results not in the index
- * keep their original snippet.
+ * from the indexed content. Index lookups run in parallel; each hit's full
+ * markdown is sent to the highlight model service, which returns the selected
+ * highlights reassembled into a single markdown document. Mutates `response` in
+ * place. Results not in the index keep their original snippet.
*/
export async function applySearchHighlights(
response: SearchV2Response,
@@ -168,39 +161,31 @@ export async function applySearchHighlights(
return { attempted, indexHits: 0, replaced: 0 };
}
- // Look up indexed markdown for every URL in parallel, then split each hit into
- // candidate spans. Keep the parsed sentences around so we can reassemble the
- // model's selected indices back into structured markdown.
+ // Look up indexed markdown for every URL in parallel, keeping the markdown for
+ // each hit so we can send it to the highlight model service.
const markdowns = await Promise.all(
targets.map(t => getIndexedMarkdownForURL(t.url, logger)),
);
const hits: {
apply: (h: string) => void;
- sentences: ReturnType<typeof parseMarkdownToSentences>;
+ markdown: string;
}[] = [];
markdowns.forEach((markdown, i) => {
if (!markdown) return;
- const sentences = parseMarkdownToSentences(markdown);
- if (sentences.length === 0) return;
- hits.push({ apply: targets[i].apply, sentences });
+ hits.push({ apply: targets[i].apply, markdown });
});
const indexHits = hits.length;
- // Score every hit in one batch call (candidate spans as explicit `lines`),
- // then for each page: budget the scored spans (neighbor + group policy) and
- // reassemble the chosen indices into a snippet.
+ // Send each hit's full markdown to the highlight model service in parallel and
+ // use the reassembled markdown it returns as the snippet.
let replaced = 0;
if (indexHits > 0) {
- const perPageSpans = await generateHighlightsBatch(
- hits.map(h => ({ query, lines: h.sentences.map(s => s.text) })),
- { logger },
+ const results = await Promise.all(
+ hits.map(h => generateHighlights(query, h.markdown, { logger })),
);
- perPageSpans.forEach((spans, i) => {
- if (!spans || spans.length === 0) return;
- const lineLengths = hits[i].sentences.map(s => s.text.length);
- const indices = selectHighlightIndices(lineLengths, spans);
- if (indices.length === 0) return;
- const snippet = assembleAnswer(hits[i].sentences, indices);
+ results.forEach((result, i) => {
+ if (!result) return;
+ const snippet = result.markdown;
if (snippet.trim() !== "") {
hits[i].apply(snippet);
replaced++;