Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

fix(api/search): use trained span pipeline for highlights beta (sentence parse + assembleAnswer)

mogery6b7249b907ed
diff --git a/apps/api/src/lib/highlight-spans.test.ts b/apps/api/src/lib/highlight-spans.test.ts
new file mode 100644
index 000000000..8e82f9ed7
--- /dev/null
+++ b/apps/api/src/lib/highlight-spans.test.ts
@@ -0,0 +1,57 @@
+import { parseMarkdownToSentences, assembleAnswer } from "./highlight-spans";
+
+describe("parseMarkdownToSentences", () => {
+ it("splits paragraphs into sentence spans and tags source", () => {
+ const s = parseMarkdownToSentences("# Title\n\nFirst. Second.");
+ expect(s).toEqual([
+ { text: "Title", source: "heading", blockId: 0 },
+ { text: "First.", source: "text", blockId: 1 },
+ { text: "Second.", source: "text", blockId: 1 },
+ ]);
+ });
+
+ it("emits a header span plus one span per table row, sharing a blockId", () => {
+ const md =
+ "| Size | Price |\n| --- | --- |\n| 128GB | $799 |\n| 256GB | $899 |";
+ const s = parseMarkdownToSentences(md);
+ expect(s).toEqual([
+ { text: "Size | Price", source: "table", blockId: 0, isHeader: true },
+ { text: "128GB | $799", source: "table", blockId: 0 },
+ { text: "256GB | $899", source: "table", blockId: 0 },
+ ]);
+ });
+});
+
+describe("assembleAnswer", () => {
+ it("rebuilds a table and auto-includes its header when only a row is selected", () => {
+ const md =
+ "| Size | Price |\n| --- | --- |\n| 128GB | $799 |\n| 256GB | $899 |";
+ const sentences = parseMarkdownToSentences(md);
+ // Select only the "256GB" row (index 2) β€” header (index 0) must be pulled in.
+ const out = assembleAnswer(sentences, [2]);
+ expect(out).toBe("| Size | Price |\n| --- | --- |\n| 256GB | $899 |");
+ });
+
+ it("merges same-block text spans back into one paragraph", () => {
+ const sentences = parseMarkdownToSentences("Alpha. Beta. Gamma.");
+ const out = assembleAnswer(sentences, [0, 2]);
+ expect(out).toBe("Alpha. Gamma.");
+ });
+
+ it("rebuilds a fenced code block from consecutive code spans", () => {
+ const md = "```js\nconst a = 1;\nconst b = 2;\n```";
+ const sentences = parseMarkdownToSentences(md);
+ const out = assembleAnswer(sentences, [0, 1]);
+ expect(out).toBe("```js\nconst a = 1;\nconst b = 2;\n```");
+ });
+
+ it("returns empty string when nothing is selected", () => {
+ const sentences = parseMarkdownToSentences("Alpha. Beta.");
+ expect(assembleAnswer(sentences, [])).toBe("");
+ });
+
+ it("ignores out-of-range indices", () => {
+ const sentences = parseMarkdownToSentences("Alpha.");
+ expect(assembleAnswer(sentences, [5, -1])).toBe("");
+ });
+});
diff --git a/apps/api/src/lib/highlight-spans.ts b/apps/api/src/lib/highlight-spans.ts
new file mode 100644
index 000000000..9e53c07b9
--- /dev/null
+++ b/apps/api/src/lib/highlight-spans.ts
@@ -0,0 +1,305 @@
+import * as marked from "marked";
+import { decode as decodeHtmlEntities } from "he";
+
+// Shared candidate-span generation + structure-aware reassembly for the
+// query-highlights models. The query-finetuned highlight model (and the
+// Fireworks gpt-oss-20b directQuote model) were trained on the line-like spans
+// `parseMarkdownToSentences` produces, so BOTH the scrape `highlights` format
+// and the search highlights beta must generate candidates here and reassemble
+// selected indices with `assembleAnswer` β€” otherwise the model scores spans it
+// never saw in training and the threshold stops meaning the same thing.
+
+type SentenceSource = "heading" | "text" | "code" | "table";
+
+interface Sentence {
+ text: string;
+ source: SentenceSource;
+ blockId: number;
+ lang?: string;
+ isHeader?: boolean;
+}
+
+function extractInlineText(tokens: marked.Token[]): string {
+ let out = "";
+ for (const t of tokens) {
+ switch (t.type) {
+ case "text":
+ out += (t as marked.Tokens.Text).tokens
+ ? extractInlineText((t as marked.Tokens.Text).tokens!)
+ : t.text;
+ break;
+ case "strong":
+ case "em":
+ case "del":
+ out += extractInlineText(
+ (t as marked.Tokens.Strong | marked.Tokens.Em | marked.Tokens.Del)
+ .tokens,
+ );
+ break;
+ case "link": {
+ const linkToken = t as marked.Tokens.Link;
+ const linkText = extractInlineText(linkToken.tokens);
+ if (
+ /^\[?\w{1,3}\]?$/.test(linkText) &&
+ linkToken.href.includes("cite_note")
+ )
+ break;
+ out += `[${linkText}](${linkToken.href})`;
+ break;
+ }
+ case "image":
+ if ((t as marked.Tokens.Image).text) {
+ out += (t as marked.Tokens.Image).text;
+ }
+ break;
+ case "codespan":
+ case "escape":
+ out += t.text;
+ break;
+ case "br":
+ out += " ";
+ break;
+ }
+ }
+ return decodeHtmlEntities(out);
+}
+
+/**
+ * Split markdown into the line-like candidate spans the highlight models were
+ * trained on. Each span carries its source kind (heading/text/code/table),
+ * a blockId grouping spans from the same source block, and table/code metadata
+ * so `assembleAnswer` can reconstruct structure from the selected spans.
+ */
+export function parseMarkdownToSentences(markdown: string): Sentence[] {
+ const result: Sentence[] = [];
+ const tokens = marked.lexer(markdown);
+ let blockId = 0;
+
+ function pushSentences(text: string, source: SentenceSource, bid: number) {
+ const trimmed = text.replace(/\s+/g, " ").trim();
+ if (!trimmed) return;
+ for (const part of trimmed.split(/(?<=[.!?])\s+/)) {
+ const s = part.trim();
+ if (s) result.push({ text: s, source, blockId: bid });
+ }
+ }
+
+ function walkBlocks(tokens: marked.Token[]) {
+ for (const token of tokens) {
+ switch (token.type) {
+ case "heading": {
+ const bid = blockId++;
+ const text = extractInlineText(
+ (token as marked.Tokens.Heading).tokens,
+ ).trim();
+ if (text) result.push({ text, source: "heading", blockId: bid });
+ break;
+ }
+ case "paragraph": {
+ const bid = blockId++;
+ pushSentences(
+ extractInlineText((token as marked.Tokens.Paragraph).tokens),
+ "text",
+ bid,
+ );
+ break;
+ }
+ case "list": {
+ for (const item of (token as marked.Tokens.List).items) {
+ walkBlocks(item.tokens);
+ }
+ break;
+ }
+ case "blockquote": {
+ walkBlocks((token as marked.Tokens.Blockquote).tokens);
+ break;
+ }
+ case "table": {
+ const bid = blockId++;
+ const table = token as marked.Tokens.Table;
+ const headerText = table.header
+ .map(cell => extractInlineText(cell.tokens).trim())
+ .filter(Boolean)
+ .join(" | ");
+ if (headerText)
+ result.push({
+ text: headerText,
+ source: "table",
+ blockId: bid,
+ isHeader: true,
+ });
+ for (const row of table.rows) {
+ const rowText = row
+ .map(cell => extractInlineText(cell.tokens).trim())
+ .filter(Boolean)
+ .join(" | ");
+ if (rowText)
+ result.push({ text: rowText, source: "table", blockId: bid });
+ }
+ break;
+ }
+ case "code": {
+ const bid = blockId++;
+ const codeToken = token as marked.Tokens.Code;
+ const lang = codeToken.lang || undefined;
+ const lines = codeToken.text.split("\n");
+ for (const line of lines) {
+ const trimmed = line.trim();
+ if (trimmed)
+ result.push({
+ text: trimmed,
+ source: "code",
+ blockId: bid,
+ lang,
+ });
+ }
+ break;
+ }
+ case "hr":
+ case "space":
+ case "html":
+ break;
+ default: {
+ const bid = blockId++;
+ if ("tokens" in token && Array.isArray(token.tokens)) {
+ walkBlocks(token.tokens);
+ } else if ("text" in token && typeof token.text === "string") {
+ pushSentences(token.text, "text", bid);
+ }
+ break;
+ }
+ }
+ }
+ }
+
+ walkBlocks(tokens);
+ return result;
+}
+
+/**
+ * Reassemble the spans the model selected (by index into the `parseMarkdown-
+ * ToSentences` output) back into readable markdown. Structure-aware: a selected
+ * table row pulls in its header and rebuilds the table syntax; consecutive code
+ * spans rebuild a fenced block; same-block text spans rejoin into a paragraph.
+ */
+export function assembleAnswer(
+ sentences: Sentence[],
+ indices: number[],
+): string {
+ const selectedIndices = new Set(
+ indices.filter(i => i >= 0 && i < sentences.length),
+ );
+
+ const tableHeaderIncluded = new Set<number>();
+ for (const i of selectedIndices) {
+ const s = sentences[i];
+ if (
+ s.source === "table" &&
+ !s.isHeader &&
+ !tableHeaderIncluded.has(s.blockId)
+ ) {
+ const headerIdx = sentences.findIndex(
+ h => h.blockId === s.blockId && h.isHeader,
+ );
+ if (headerIdx !== -1) {
+ selectedIndices.add(headerIdx);
+ tableHeaderIncluded.add(s.blockId);
+ }
+ }
+ }
+
+ const selected = [...selectedIndices]
+ .sort((a, b) => a - b)
+ .map(i => ({ index: i, ...sentences[i] }));
+
+ const parts: string[] = [];
+ let codeRun: { texts: string[]; lang?: string } | null = null;
+
+ function flushCode() {
+ if (!codeRun) return;
+ const fence = codeRun.lang ? "```" + codeRun.lang : "```";
+ parts.push(fence + "\n" + codeRun.texts.join("\n") + "\n```");
+ codeRun = null;
+ }
+
+ let tableRows: string[] | null = null;
+
+ function formatTableRow(text: string): string {
+ return "| " + text.split(" | ").join(" | ") + " |";
+ }
+
+ function startTable(cur: (typeof selected)[number]) {
+ tableRows = [];
+ if (cur.isHeader) {
+ const cols = cur.text.split(" | ");
+ const escaped = cols.map(c => c.replace(/\*/g, "\\*"));
+ tableRows.push("| " + escaped.join(" | ") + " |");
+ tableRows.push("| " + cols.map(() => "---").join(" | ") + " |");
+ } else {
+ tableRows.push(formatTableRow(cur.text));
+ }
+ }
+
+ function appendTableRow(cur: (typeof selected)[number]) {
+ if (!tableRows) {
+ startTable(cur);
+ return;
+ }
+ if (cur.isHeader) {
+ const cols = cur.text.split(" | ");
+ const escaped = cols.map(c => c.replace(/\*/g, "\\*"));
+ tableRows.unshift("| " + escaped.join(" | ") + " |");
+ tableRows.splice(1, 0, "| " + cols.map(() => "---").join(" | ") + " |");
+ } else {
+ tableRows.push(formatTableRow(cur.text));
+ }
+ }
+
+ function flushTable() {
+ if (!tableRows) return;
+ parts.push(tableRows.join("\n"));
+ tableRows = null;
+ }
+
+ for (let k = 0; k < selected.length; k++) {
+ const cur = selected[k];
+ const prev = k > 0 ? selected[k - 1] : null;
+ const samePrevBlock =
+ prev && prev.blockId === cur.blockId && prev.source === cur.source;
+
+ if (cur.source === "code") {
+ flushTable();
+ if (samePrevBlock && codeRun) {
+ codeRun.texts.push(cur.text);
+ } else {
+ flushCode();
+ codeRun = { texts: [cur.text], lang: cur.lang };
+ }
+ continue;
+ }
+
+ if (cur.source === "table") {
+ flushCode();
+ if (samePrevBlock && tableRows) {
+ appendTableRow(cur);
+ } else {
+ flushTable();
+ startTable(cur);
+ }
+ continue;
+ }
+
+ flushCode();
+ flushTable();
+
+ if (samePrevBlock) {
+ parts[parts.length - 1] += " " + cur.text;
+ } else {
+ parts.push(cur.text);
+ }
+ }
+
+ flushCode();
+ flushTable();
+ return parts.join("\n\n");
+}
diff --git a/apps/api/src/scraper/scrapeURL/transformers/query.ts b/apps/api/src/scraper/scrapeURL/transformers/query.ts
index ca09d6981..2449b38b1 100644
--- a/apps/api/src/scraper/scrapeURL/transformers/query.ts
+++ b/apps/api/src/scraper/scrapeURL/transformers/query.ts
@@ -1,297 +1,19 @@
import { generateText } from "ai";
-import * as marked from "marked";
-import { decode as decodeHtmlEntities } from "he";
import { Document, FormatObject } from "../../../controllers/v2/types";
import { Meta } from "..";
import { getModel } from "../../../lib/generic-ai";
import { hasFormatOfType } from "../../../lib/format-utils";
import { calculateCost } from "./llmExtract";
+import {
+ parseMarkdownToSentences,
+ assembleAnswer,
+} from "../../../lib/highlight-spans";
const PROMPT_TAGS = /(<\/?)(query|page|lines)([\s>])/gi;
function escapePromptTags(text: string): string {
return text.replace(PROMPT_TAGS, "$1\u200B$2$3");
}
-type SentenceSource = "heading" | "text" | "code" | "table";
-
-interface Sentence {
- text: string;
- source: SentenceSource;
- blockId: number;
- lang?: string;
- isHeader?: boolean;
-}
-
-function extractInlineText(tokens: marked.Token[]): string {
- let out = "";
- for (const t of tokens) {
- switch (t.type) {
- case "text":
- out += (t as marked.Tokens.Text).tokens
- ? extractInlineText((t as marked.Tokens.Text).tokens!)
- : t.text;
- break;
- case "strong":
- case "em":
- case "del":
- out += extractInlineText(
- (t as marked.Tokens.Strong | marked.Tokens.Em | marked.Tokens.Del)
- .tokens,
- );
- break;
- case "link": {
- const linkToken = t as marked.Tokens.Link;
- const linkText = extractInlineText(linkToken.tokens);
- if (
- /^\[?\w{1,3}\]?$/.test(linkText) &&
- linkToken.href.includes("cite_note")
- )
- break;
- out += `[${linkText}](${linkToken.href})`;
- break;
- }
- case "image":
- if ((t as marked.Tokens.Image).text) {
- out += (t as marked.Tokens.Image).text;
- }
- break;
- case "codespan":
- case "escape":
- out += t.text;
- break;
- case "br":
- out += " ";
- break;
- }
- }
- return decodeHtmlEntities(out);
-}
-
-function parseMarkdownToSentences(markdown: string): Sentence[] {
- const result: Sentence[] = [];
- const tokens = marked.lexer(markdown);
- let blockId = 0;
-
- function pushSentences(text: string, source: SentenceSource, bid: number) {
- const trimmed = text.replace(/\s+/g, " ").trim();
- if (!trimmed) return;
- for (const part of trimmed.split(/(?<=[.!?])\s+/)) {
- const s = part.trim();
- if (s) result.push({ text: s, source, blockId: bid });
- }
- }
-
- function walkBlocks(tokens: marked.Token[]) {
- for (const token of tokens) {
- switch (token.type) {
- case "heading": {
- const bid = blockId++;
- const text = extractInlineText(
- (token as marked.Tokens.Heading).tokens,
- ).trim();
- if (text) result.push({ text, source: "heading", blockId: bid });
- break;
- }
- case "paragraph": {
- const bid = blockId++;
- pushSentences(
- extractInlineText((token as marked.Tokens.Paragraph).tokens),
- "text",
- bid,
- );
- break;
- }
- case "list": {
- for (const item of (token as marked.Tokens.List).items) {
- walkBlocks(item.tokens);
- }
- break;
- }
- case "blockquote": {
- walkBlocks((token as marked.Tokens.Blockquote).tokens);
- break;
- }
- case "table": {
- const bid = blockId++;
- const table = token as marked.Tokens.Table;
- const headerText = table.header
- .map(cell => extractInlineText(cell.tokens).trim())
- .filter(Boolean)
- .join(" | ");
- if (headerText)
- result.push({
- text: headerText,
- source: "table",
- blockId: bid,
- isHeader: true,
- });
- for (const row of table.rows) {
- const rowText = row
- .map(cell => extractInlineText(cell.tokens).trim())
- .filter(Boolean)
- .join(" | ");
- if (rowText)
- result.push({ text: rowText, source: "table", blockId: bid });
- }
- break;
- }
- case "code": {
- const bid = blockId++;
- const codeToken = token as marked.Tokens.Code;
- const lang = codeToken.lang || undefined;
- const lines = codeToken.text.split("\n");
- for (const line of lines) {
- const trimmed = line.trim();
- if (trimmed)
- result.push({
- text: trimmed,
- source: "code",
- blockId: bid,
- lang,
- });
- }
- break;
- }
- case "hr":
- case "space":
- case "html":
- break;
- default: {
- const bid = blockId++;
- if ("tokens" in token && Array.isArray(token.tokens)) {
- walkBlocks(token.tokens);
- } else if ("text" in token && typeof token.text === "string") {
- pushSentences(token.text, "text", bid);
- }
- break;
- }
- }
- }
- }
-
- walkBlocks(tokens);
- return result;
-}
-
-function assembleAnswer(sentences: Sentence[], indices: number[]): string {
- const selectedIndices = new Set(
- indices.filter(i => i >= 0 && i < sentences.length),
- );
-
- const tableHeaderIncluded = new Set<number>();
- for (const i of selectedIndices) {
- const s = sentences[i];
- if (
- s.source === "table" &&
- !s.isHeader &&
- !tableHeaderIncluded.has(s.blockId)
- ) {
- const headerIdx = sentences.findIndex(
- h => h.blockId === s.blockId && h.isHeader,
- );
- if (headerIdx !== -1) {
- selectedIndices.add(headerIdx);
- tableHeaderIncluded.add(s.blockId);
- }
- }
- }
-
- const selected = [...selectedIndices]
- .sort((a, b) => a - b)
- .map(i => ({ index: i, ...sentences[i] }));
-
- const parts: string[] = [];
- let codeRun: { texts: string[]; lang?: string } | null = null;
-
- function flushCode() {
- if (!codeRun) return;
- const fence = codeRun.lang ? "```" + codeRun.lang : "```";
- parts.push(fence + "\n" + codeRun.texts.join("\n") + "\n```");
- codeRun = null;
- }
-
- let tableRows: string[] | null = null;
-
- function formatTableRow(text: string): string {
- return "| " + text.split(" | ").join(" | ") + " |";
- }
-
- function startTable(cur: (typeof selected)[number]) {
- tableRows = [];
- if (cur.isHeader) {
- const cols = cur.text.split(" | ");
- const escaped = cols.map(c => c.replace(/\*/g, "\\*"));
- tableRows.push("| " + escaped.join(" | ") + " |");
- tableRows.push("| " + cols.map(() => "---").join(" | ") + " |");
- } else {
- tableRows.push(formatTableRow(cur.text));
- }
- }
-
- function appendTableRow(cur: (typeof selected)[number]) {
- if (!tableRows) {
- startTable(cur);
- return;
- }
- if (cur.isHeader) {
- const cols = cur.text.split(" | ");
- const escaped = cols.map(c => c.replace(/\*/g, "\\*"));
- tableRows.unshift("| " + escaped.join(" | ") + " |");
- tableRows.splice(1, 0, "| " + cols.map(() => "---").join(" | ") + " |");
- } else {
- tableRows.push(formatTableRow(cur.text));
- }
- }
-
- function flushTable() {
- if (!tableRows) return;
- parts.push(tableRows.join("\n"));
- tableRows = null;
- }
-
- for (let k = 0; k < selected.length; k++) {
- const cur = selected[k];
- const prev = k > 0 ? selected[k - 1] : null;
- const samePrevBlock =
- prev && prev.blockId === cur.blockId && prev.source === cur.source;
-
- if (cur.source === "code") {
- flushTable();
- if (samePrevBlock && codeRun) {
- codeRun.texts.push(cur.text);
- } else {
- flushCode();
- codeRun = { texts: [cur.text], lang: cur.lang };
- }
- continue;
- }
-
- if (cur.source === "table") {
- flushCode();
- if (samePrevBlock && tableRows) {
- appendTableRow(cur);
- } else {
- flushTable();
- startTable(cur);
- }
- continue;
- }
-
- flushCode();
- flushTable();
-
- if (samePrevBlock) {
- parts[parts.length - 1] += " " + cur.text;
- } else {
- parts.push(cur.text);
- }
- }
-
- flushCode();
- flushTable();
- return parts.join("\n\n");
-}
-
const DIRECT_QUOTE_MODEL = {
id: "accounts/thomas-bfc570/models/gpt-oss-20b-query-finetune-2026-04-15#accounts/thomas-bfc570/deployments/gpt-oss-20b-query-finetune-2026-04-24",
provider: "fireworks" as const,
diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.ts
index 82cbe6678..8e6b02a35 100644
--- a/apps/api/src/search/highlight-model.test.ts
+++ b/apps/api/src/search/highlight-model.test.ts
@@ -38,18 +38,15 @@ describe("generateHighlightsBatch", () => {
expect(fetchMock).not.toHaveBeenCalled();
});
- it("posts every page to /batch_highlight with the bearer token", async () => {
+ it("posts each page's spans as `lines` to /batch_highlight with the bearer token", async () => {
const fetchMock = mockFetchOnce({
- results: [
- { pruned_markdown: "Highlight A\n" },
- { pruned_markdown: "Highlight B\n" },
- ],
+ results: [{ highlights: [] }, { highlights: [] }],
});
await generateHighlightsBatch(
[
- { query: "q1", markdown: "page one" },
- { query: "q2", markdown: "page two" },
+ { query: "q1", lines: ["a one", "b one"] },
+ { query: "q2", lines: ["a two"] },
],
{ logger },
);
@@ -62,53 +59,73 @@ describe("generateHighlightsBatch", () => {
const sent = JSON.parse(init.body);
expect(sent.requests).toHaveLength(2);
- expect(sent.requests[0]).toMatchObject({ query: "q1", markdown: "page one" });
+ expect(sent.requests[0]).toMatchObject({
+ query: "q1",
+ lines: ["a one", "b one"],
+ });
// Threshold + budget are always sent so the cutoff matches the trained format.
expect(typeof sent.requests[0].threshold).toBe("number");
expect(typeof sent.requests[0].max_highlight_chars).toBe("number");
});
- it("maps pruned_markdown back to each item, trimmed, aligned by index", async () => {
+ it("returns the selected span indices per page, aligned by request order", async () => {
mockFetchOnce({
results: [
- { pruned_markdown: "First page highlight\n" },
- { pruned_markdown: "Second page highlight\nmore\n" },
+ {
+ highlights: [
+ { index: 0, score: 0.9 },
+ { index: 2, score: 0.5 },
+ ],
+ },
+ { highlights: [{ index: 1, score: 0.7 }] },
],
});
const out = await generateHighlightsBatch(
[
- { query: "q1", markdown: "a" },
- { query: "q2", markdown: "b" },
+ { query: "q1", lines: ["l0", "l1", "l2"] },
+ { query: "q2", lines: ["l0", "l1"] },
+ ],
+ { logger },
+ );
+
+ expect(out).toEqual([[0, 2], [1]]);
+ });
+
+ it("returns an empty array for a page with no selected spans", async () => {
+ mockFetchOnce({ results: [{ highlights: [] }, {}] });
+
+ const out = await generateHighlightsBatch(
+ [
+ { query: "q", lines: ["a"] },
+ { query: "q", lines: ["b"] },
],
{ logger },
);
- expect(out).toEqual([
- "First page highlight",
- "Second page highlight\nmore",
- ]);
+ expect(out).toEqual([[], []]);
});
- it("returns null for empty / missing pruned_markdown entries", async () => {
+ it("ignores malformed (non-integer / missing) indices", async () => {
mockFetchOnce({
results: [
- { pruned_markdown: "" },
- { pruned_markdown: " \n" },
- {},
+ {
+ highlights: [
+ { index: 1, score: 0.9 },
+ { score: 0.8 }, // missing index
+ { index: 2.5, score: 0.7 }, // non-integer
+ { index: 3, score: 0.6 },
+ ],
+ },
],
});
const out = await generateHighlightsBatch(
- [
- { query: "q", markdown: "a" },
- { query: "q", markdown: "b" },
- { query: "q", markdown: "c" },
- ],
+ [{ query: "q", lines: ["a", "b", "c", "d"] }],
{ logger },
);
- expect(out).toEqual([null, null, null]);
+ expect(out).toEqual([[1, 3]]);
});
it("falls back to nulls (one per item) when the service errors", async () => {
@@ -116,8 +133,8 @@ describe("generateHighlightsBatch", () => {
const out = await generateHighlightsBatch(
[
- { query: "q", markdown: "a" },
- { query: "q", markdown: "b" },
+ { query: "q", lines: ["a"] },
+ { query: "q", lines: ["b"] },
],
{ logger },
);
diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.ts
index e41a192db..9dd43cfaf 100644
--- a/apps/api/src/search/highlight-model.ts
+++ b/apps/api/src/search/highlight-model.ts
@@ -1,58 +1,68 @@
import type { Logger } from "winston";
import { config } from "../config";
-// Query Highlights model service: a query-finetuned span scorer (no LLM). It
-// splits each page's markdown into line-like candidate spans β€” the same shape
-// the model was trained on β€” scores every span against the query, and returns
-// the selected spans concatenated in page order as `pruned_markdown`. The
-// service keeps a resident GPU model, so we send every page through one
-// `/batch_highlight` call instead of one request per page. Endpoint + token are
+// Query Highlights model service: a query-finetuned span scorer (no LLM). It was
+// trained on the line-like spans `parseMarkdownToSentences` produces, so callers
+// pass those spans as explicit `lines`, the service scores each against the
+// query, and returns the selected spans as `highlights[]` with their `index`
+// back into the input `lines` array. Callers map those indices through
+// `assembleAnswer` to rebuild structure (tables/code). We push every page
+// through one `/batch_highlight` call (resident GPU model). Endpoint + token are
// config-gated (HIGHLIGHT_MODEL_URL / HIGHLIGHT_MODEL_TOKEN); callers must
// confirm they're set via highlightsEnvReady() before invoking.
// Experimental score cutoff for keeping a span, tuned against the line-span
// format the model was trained on.
const HIGHLIGHT_THRESHOLD = 0.08;
-// Character budget for the assembled snippet; the service keeps the
-// highest-scoring spans until the budget is reached.
+// Character budget per page; the service keeps the highest-scoring spans until
+// the budget is reached (measured on raw span text).
const MAX_HIGHLIGHT_CHARS = 800;
const REQUEST_TIMEOUT_MS = 30000;
interface HighlightItem {
query: string;
- markdown: string;
+ // Candidate spans for the page, in document order β€” from
+ // parseMarkdownToSentences(markdown).map(s => s.text).
+ lines: string[];
+}
+
+interface HighlightSpan {
+ index?: number;
+ score?: number;
}
interface HighlightResult {
- pruned_markdown?: string;
+ highlights?: HighlightSpan[];
}
interface BatchHighlightResponse {
results?: HighlightResult[];
}
-// The service returns the selected spans joined with "\n" (often with a trailing
-// newline). Trim to a clean snippet; null when nothing was selected.
-function prunedToSnippet(result: HighlightResult | undefined): string | null {
- const pruned = result?.pruned_markdown;
- if (typeof pruned !== "string") return null;
- const trimmed = pruned.trim();
- return trimmed === "" ? null : trimmed;
+// Pull the selected span indices out of one page's result. The service already
+// applied the threshold / char budget, so we just collect the kept indices.
+function selectedIndices(result: HighlightResult | undefined): number[] {
+ const highlights = result?.highlights ?? [];
+ const out: number[] = [];
+ for (const h of highlights) {
+ if (typeof h.index === "number" && Number.isInteger(h.index)) {
+ out.push(h.index);
+ }
+ }
+ return out;
}
/**
- * Generate query-relevant highlights for many pages in a single batch call to
- * the query-highlights model. Each page's markdown is sent as-is; the service
- * splits it into the line-like spans used in training, scores them against the
- * query, and returns the selected spans joined in page order
- * (`pruned_markdown`). Returns an array aligned to `items`: the highlight
- * snippet for each page, or null when nothing clears the threshold (or the whole
- * batch fails β€” highlights are best-effort, callers keep the provider snippet).
+ * Score many pages' candidate spans in a single batch call to the
+ * query-highlights model. Returns an array aligned to `items`: for each page,
+ * the indices (into that page's `lines`) the model selected, or null when the
+ * whole batch fails (so callers can keep the provider snippet). A page with no
+ * spans clearing the threshold yields an empty array, not null.
*/
export async function generateHighlightsBatch(
items: HighlightItem[],
opts: { logger: Logger },
-): Promise<(string | null)[]> {
+): Promise<(number[] | null)[]> {
if (items.length === 0) return [];
const controller = new AbortController();
@@ -68,7 +78,7 @@ export async function generateHighlightsBatch(
body: JSON.stringify({
requests: items.map(item => ({
query: item.query,
- markdown: item.markdown,
+ lines: item.lines,
threshold: HIGHLIGHT_THRESHOLD,
max_highlight_chars: MAX_HIGHLIGHT_CHARS,
})),
@@ -85,15 +95,15 @@ export async function generateHighlightsBatch(
const data = (await res.json()) as BatchHighlightResponse;
const results = data.results ?? [];
- const snippets = items.map((_, i) => prunedToSnippet(results[i]));
+ const indices = items.map((_, i) => selectedIndices(results[i]));
opts.logger.info("query highlights batch generated", {
pages: items.length,
- withHighlights: snippets.filter(Boolean).length,
+ withHighlights: indices.filter(x => x.length > 0).length,
elapsedMs: Date.now() - start,
});
- return snippets;
+ return indices;
} catch (error) {
opts.logger.warn("query highlights batch failed", {
error: error instanceof Error ? error.message : String(error),
diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts
index 9a6eb0a9f..116f31974 100644
--- a/apps/api/src/search/highlights.ts
+++ b/apps/api/src/search/highlights.ts
@@ -11,6 +11,10 @@ import { parseMarkdown } from "../lib/html-to-markdown";
import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";
import type { ScrapeOptions } from "../controllers/v2/types";
import { generateHighlightsBatch } from "./highlight-model";
+import {
+ parseMarkdownToSentences,
+ assembleAnswer,
+} from "../lib/highlight-spans";
import { config } from "../config";
// How far back into the index we're willing to reach for highlight source text.
@@ -122,9 +126,12 @@ async function getIndexedMarkdownForURL(
/**
* For each search result: look up the URL in our index (last 30 days), and if
* present, replace the provider snippet with query-relevant highlights generated
- * from the indexed content. Index lookups run in parallel, then every hit is
- * scored in a single batch call to the query-highlights model. Mutates
- * `response` in place. Results not in the index keep their original snippet.
+ * from the indexed content. Index lookups run in parallel; each hit's markdown
+ * is split into the same candidate spans the highlight model was trained on
+ * (parseMarkdownToSentences), every hit is scored in a single batch call, and
+ * the selected span indices are reassembled with assembleAnswer (structure-aware
+ * β€” rebuilds tables/code). Mutates `response` in place. Results not in the index
+ * keep their original snippet.
*/
export async function applySearchHighlights(
response: SearchV2Response,
@@ -160,25 +167,36 @@ export async function applySearchHighlights(
return { attempted, indexHits: 0, replaced: 0 };
}
- // Look up indexed markdown for every URL in parallel, keeping only the hits.
+ // Look up indexed markdown for every URL in parallel, then split each hit into
+ // candidate spans. Keep the parsed sentences around so we can reassemble the
+ // model's selected indices back into structured markdown.
const markdowns = await Promise.all(
targets.map(t => getIndexedMarkdownForURL(t.url, logger)),
);
- const hits: { apply: (h: string) => void; markdown: string }[] = [];
+ const hits: {
+ apply: (h: string) => void;
+ sentences: ReturnType<typeof parseMarkdownToSentences>;
+ }[] = [];
markdowns.forEach((markdown, i) => {
- if (markdown) hits.push({ apply: targets[i].apply, markdown });
+ if (!markdown) return;
+ const sentences = parseMarkdownToSentences(markdown);
+ if (sentences.length === 0) return;
+ hits.push({ apply: targets[i].apply, sentences });
});
const indexHits = hits.length;
- // Score every hit in one batch call, then apply the highlights that came back.
+ // Score every hit in one batch call (candidate spans as explicit `lines`),
+ // then reassemble each page's selected span indices into a snippet.
let replaced = 0;
if (indexHits > 0) {
- const snippets = await generateHighlightsBatch(
- hits.map(h => ({ query, markdown: h.markdown })),
+ const perPageIndices = await generateHighlightsBatch(
+ hits.map(h => ({ query, lines: h.sentences.map(s => s.text) })),
{ logger },
);
- snippets.forEach((snippet, i) => {
- if (snippet && snippet.trim() !== "") {
+ perPageIndices.forEach((indices, i) => {
+ if (!indices || indices.length === 0) return;
+ const snippet = assembleAnswer(hits[i].sentences, indices);
+ if (snippet.trim() !== "") {
hits[i].apply(snippet);
replaced++;
}