Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/search): use trained span pipeline for highlights beta (sentence parse + assembleAnswer)
mogery6b7249b907ed
diff --git a/apps/api/src/lib/highlight-spans.test.ts b/apps/api/src/lib/highlight-spans.test.tsnew file mode 100644index 000000000..8e82f9ed7--- /dev/null+++ b/apps/api/src/lib/highlight-spans.test.ts@@ -0,0 +1,57 @@+import { parseMarkdownToSentences, assembleAnswer } from "./highlight-spans";++describe("parseMarkdownToSentences", () => {+ it("splits paragraphs into sentence spans and tags source", () => {+ const s = parseMarkdownToSentences("# Title\n\nFirst. Second.");+ expect(s).toEqual([+ { text: "Title", source: "heading", blockId: 0 },+ { text: "First.", source: "text", blockId: 1 },+ { text: "Second.", source: "text", blockId: 1 },+ ]);+ });++ it("emits a header span plus one span per table row, sharing a blockId", () => {+ const md =+ "| Size | Price |\n| --- | --- |\n| 128GB | $799 |\n| 256GB | $899 |";+ const s = parseMarkdownToSentences(md);+ expect(s).toEqual([+ { text: "Size | Price", source: "table", blockId: 0, isHeader: true },+ { text: "128GB | $799", source: "table", blockId: 0 },+ { text: "256GB | $899", source: "table", blockId: 0 },+ ]);+ });+});++describe("assembleAnswer", () => {+ it("rebuilds a table and auto-includes its header when only a row is selected", () => {+ const md =+ "| Size | Price |\n| --- | --- |\n| 128GB | $799 |\n| 256GB | $899 |";+ const sentences = parseMarkdownToSentences(md);+ // Select only the "256GB" row (index 2) β header (index 0) must be pulled in.+ const out = assembleAnswer(sentences, [2]);+ expect(out).toBe("| Size | Price |\n| --- | --- |\n| 256GB | $899 |");+ });++ it("merges same-block text spans back into one paragraph", () => {+ const sentences = parseMarkdownToSentences("Alpha. Beta. Gamma.");+ const out = assembleAnswer(sentences, [0, 2]);+ expect(out).toBe("Alpha. Gamma.");+ });++ it("rebuilds a fenced code block from consecutive code spans", () => {+ const md = "```js\nconst a = 1;\nconst b = 2;\n```";+ const sentences = parseMarkdownToSentences(md);+ const out = assembleAnswer(sentences, [0, 1]);+ expect(out).toBe("```js\nconst a = 1;\nconst b = 2;\n```");+ });++ it("returns empty string when nothing is selected", () => {+ const sentences = parseMarkdownToSentences("Alpha. Beta.");+ expect(assembleAnswer(sentences, [])).toBe("");+ });++ it("ignores out-of-range indices", () => {+ const sentences = parseMarkdownToSentences("Alpha.");+ expect(assembleAnswer(sentences, [5, -1])).toBe("");+ });+});diff --git a/apps/api/src/lib/highlight-spans.ts b/apps/api/src/lib/highlight-spans.tsnew file mode 100644index 000000000..9e53c07b9--- /dev/null+++ b/apps/api/src/lib/highlight-spans.ts@@ -0,0 +1,305 @@+import * as marked from "marked";+import { decode as decodeHtmlEntities } from "he";++// Shared candidate-span generation + structure-aware reassembly for the+// query-highlights models. The query-finetuned highlight model (and the+// Fireworks gpt-oss-20b directQuote model) were trained on the line-like spans+// `parseMarkdownToSentences` produces, so BOTH the scrape `highlights` format+// and the search highlights beta must generate candidates here and reassemble+// selected indices with `assembleAnswer` β otherwise the model scores spans it+// never saw in training and the threshold stops meaning the same thing.++type SentenceSource = "heading" | "text" | "code" | "table";++interface Sentence {+ text: string;+ source: SentenceSource;+ blockId: number;+ lang?: string;+ isHeader?: boolean;+}++function extractInlineText(tokens: marked.Token[]): string {+ let out = "";+ for (const t of tokens) {+ switch (t.type) {+ case "text":+ out += (t as marked.Tokens.Text).tokens+ ? extractInlineText((t as marked.Tokens.Text).tokens!)+ : t.text;+ break;+ case "strong":+ case "em":+ case "del":+ out += extractInlineText(+ (t as marked.Tokens.Strong | marked.Tokens.Em | marked.Tokens.Del)+ .tokens,+ );+ break;+ case "link": {+ const linkToken = t as marked.Tokens.Link;+ const linkText = extractInlineText(linkToken.tokens);+ if (+ /^\[?\w{1,3}\]?$/.test(linkText) &&+ linkToken.href.includes("cite_note")+ )+ break;+ out += `[${linkText}](${linkToken.href})`;+ break;+ }+ case "image":+ if ((t as marked.Tokens.Image).text) {+ out += (t as marked.Tokens.Image).text;+ }+ break;+ case "codespan":+ case "escape":+ out += t.text;+ break;+ case "br":+ out += " ";+ break;+ }+ }+ return decodeHtmlEntities(out);+}++/**+ * Split markdown into the line-like candidate spans the highlight models were+ * trained on. Each span carries its source kind (heading/text/code/table),+ * a blockId grouping spans from the same source block, and table/code metadata+ * so `assembleAnswer` can reconstruct structure from the selected spans.+ */+export function parseMarkdownToSentences(markdown: string): Sentence[] {+ const result: Sentence[] = [];+ const tokens = marked.lexer(markdown);+ let blockId = 0;++ function pushSentences(text: string, source: SentenceSource, bid: number) {+ const trimmed = text.replace(/\s+/g, " ").trim();+ if (!trimmed) return;+ for (const part of trimmed.split(/(?<=[.!?])\s+/)) {+ const s = part.trim();+ if (s) result.push({ text: s, source, blockId: bid });+ }+ }++ function walkBlocks(tokens: marked.Token[]) {+ for (const token of tokens) {+ switch (token.type) {+ case "heading": {+ const bid = blockId++;+ const text = extractInlineText(+ (token as marked.Tokens.Heading).tokens,+ ).trim();+ if (text) result.push({ text, source: "heading", blockId: bid });+ break;+ }+ case "paragraph": {+ const bid = blockId++;+ pushSentences(+ extractInlineText((token as marked.Tokens.Paragraph).tokens),+ "text",+ bid,+ );+ break;+ }+ case "list": {+ for (const item of (token as marked.Tokens.List).items) {+ walkBlocks(item.tokens);+ }+ break;+ }+ case "blockquote": {+ walkBlocks((token as marked.Tokens.Blockquote).tokens);+ break;+ }+ case "table": {+ const bid = blockId++;+ const table = token as marked.Tokens.Table;+ const headerText = table.header+ .map(cell => extractInlineText(cell.tokens).trim())+ .filter(Boolean)+ .join(" | ");+ if (headerText)+ result.push({+ text: headerText,+ source: "table",+ blockId: bid,+ isHeader: true,+ });+ for (const row of table.rows) {+ const rowText = row+ .map(cell => extractInlineText(cell.tokens).trim())+ .filter(Boolean)+ .join(" | ");+ if (rowText)+ result.push({ text: rowText, source: "table", blockId: bid });+ }+ break;+ }+ case "code": {+ const bid = blockId++;+ const codeToken = token as marked.Tokens.Code;+ const lang = codeToken.lang || undefined;+ const lines = codeToken.text.split("\n");+ for (const line of lines) {+ const trimmed = line.trim();+ if (trimmed)+ result.push({+ text: trimmed,+ source: "code",+ blockId: bid,+ lang,+ });+ }+ break;+ }+ case "hr":+ case "space":+ case "html":+ break;+ default: {+ const bid = blockId++;+ if ("tokens" in token && Array.isArray(token.tokens)) {+ walkBlocks(token.tokens);+ } else if ("text" in token && typeof token.text === "string") {+ pushSentences(token.text, "text", bid);+ }+ break;+ }+ }+ }+ }++ walkBlocks(tokens);+ return result;+}++/**+ * Reassemble the spans the model selected (by index into the `parseMarkdown-+ * ToSentences` output) back into readable markdown. Structure-aware: a selected+ * table row pulls in its header and rebuilds the table syntax; consecutive code+ * spans rebuild a fenced block; same-block text spans rejoin into a paragraph.+ */+export function assembleAnswer(+ sentences: Sentence[],+ indices: number[],+): string {+ const selectedIndices = new Set(+ indices.filter(i => i >= 0 && i < sentences.length),+ );++ const tableHeaderIncluded = new Set<number>();+ for (const i of selectedIndices) {+ const s = sentences[i];+ if (+ s.source === "table" &&+ !s.isHeader &&+ !tableHeaderIncluded.has(s.blockId)+ ) {+ const headerIdx = sentences.findIndex(+ h => h.blockId === s.blockId && h.isHeader,+ );+ if (headerIdx !== -1) {+ selectedIndices.add(headerIdx);+ tableHeaderIncluded.add(s.blockId);+ }+ }+ }++ const selected = [...selectedIndices]+ .sort((a, b) => a - b)+ .map(i => ({ index: i, ...sentences[i] }));++ const parts: string[] = [];+ let codeRun: { texts: string[]; lang?: string } | null = null;++ function flushCode() {+ if (!codeRun) return;+ const fence = codeRun.lang ? "```" + codeRun.lang : "```";+ parts.push(fence + "\n" + codeRun.texts.join("\n") + "\n```");+ codeRun = null;+ }++ let tableRows: string[] | null = null;++ function formatTableRow(text: string): string {+ return "| " + text.split(" | ").join(" | ") + " |";+ }++ function startTable(cur: (typeof selected)[number]) {+ tableRows = [];+ if (cur.isHeader) {+ const cols = cur.text.split(" | ");+ const escaped = cols.map(c => c.replace(/\*/g, "\\*"));+ tableRows.push("| " + escaped.join(" | ") + " |");+ tableRows.push("| " + cols.map(() => "---").join(" | ") + " |");+ } else {+ tableRows.push(formatTableRow(cur.text));+ }+ }++ function appendTableRow(cur: (typeof selected)[number]) {+ if (!tableRows) {+ startTable(cur);+ return;+ }+ if (cur.isHeader) {+ const cols = cur.text.split(" | ");+ const escaped = cols.map(c => c.replace(/\*/g, "\\*"));+ tableRows.unshift("| " + escaped.join(" | ") + " |");+ tableRows.splice(1, 0, "| " + cols.map(() => "---").join(" | ") + " |");+ } else {+ tableRows.push(formatTableRow(cur.text));+ }+ }++ function flushTable() {+ if (!tableRows) return;+ parts.push(tableRows.join("\n"));+ tableRows = null;+ }++ for (let k = 0; k < selected.length; k++) {+ const cur = selected[k];+ const prev = k > 0 ? selected[k - 1] : null;+ const samePrevBlock =+ prev && prev.blockId === cur.blockId && prev.source === cur.source;++ if (cur.source === "code") {+ flushTable();+ if (samePrevBlock && codeRun) {+ codeRun.texts.push(cur.text);+ } else {+ flushCode();+ codeRun = { texts: [cur.text], lang: cur.lang };+ }+ continue;+ }++ if (cur.source === "table") {+ flushCode();+ if (samePrevBlock && tableRows) {+ appendTableRow(cur);+ } else {+ flushTable();+ startTable(cur);+ }+ continue;+ }++ flushCode();+ flushTable();++ if (samePrevBlock) {+ parts[parts.length - 1] += " " + cur.text;+ } else {+ parts.push(cur.text);+ }+ }++ flushCode();+ flushTable();+ return parts.join("\n\n");+}diff --git a/apps/api/src/scraper/scrapeURL/transformers/query.ts b/apps/api/src/scraper/scrapeURL/transformers/query.tsindex ca09d6981..2449b38b1 100644--- a/apps/api/src/scraper/scrapeURL/transformers/query.ts+++ b/apps/api/src/scraper/scrapeURL/transformers/query.ts@@ -1,297 +1,19 @@import { generateText } from "ai";-import * as marked from "marked";-import { decode as decodeHtmlEntities } from "he";import { Document, FormatObject } from "../../../controllers/v2/types";import { Meta } from "..";import { getModel } from "../../../lib/generic-ai";import { hasFormatOfType } from "../../../lib/format-utils";import { calculateCost } from "./llmExtract";+import {+ parseMarkdownToSentences,+ assembleAnswer,+} from "../../../lib/highlight-spans";const PROMPT_TAGS = /(<\/?)(query|page|lines)([\s>])/gi;function escapePromptTags(text: string): string {return text.replace(PROMPT_TAGS, "$1\u200B$2$3");}-type SentenceSource = "heading" | "text" | "code" | "table";--interface Sentence {- text: string;- source: SentenceSource;- blockId: number;- lang?: string;- isHeader?: boolean;-}--function extractInlineText(tokens: marked.Token[]): string {- let out = "";- for (const t of tokens) {- switch (t.type) {- case "text":- out += (t as marked.Tokens.Text).tokens- ? extractInlineText((t as marked.Tokens.Text).tokens!)- : t.text;- break;- case "strong":- case "em":- case "del":- out += extractInlineText(- (t as marked.Tokens.Strong | marked.Tokens.Em | marked.Tokens.Del)- .tokens,- );- break;- case "link": {- const linkToken = t as marked.Tokens.Link;- const linkText = extractInlineText(linkToken.tokens);- if (- /^\[?\w{1,3}\]?$/.test(linkText) &&- linkToken.href.includes("cite_note")- )- break;- out += `[${linkText}](${linkToken.href})`;- break;- }- case "image":- if ((t as marked.Tokens.Image).text) {- out += (t as marked.Tokens.Image).text;- }- break;- case "codespan":- case "escape":- out += t.text;- break;- case "br":- out += " ";- break;- }- }- return decodeHtmlEntities(out);-}--function parseMarkdownToSentences(markdown: string): Sentence[] {- const result: Sentence[] = [];- const tokens = marked.lexer(markdown);- let blockId = 0;-- function pushSentences(text: string, source: SentenceSource, bid: number) {- const trimmed = text.replace(/\s+/g, " ").trim();- if (!trimmed) return;- for (const part of trimmed.split(/(?<=[.!?])\s+/)) {- const s = part.trim();- if (s) result.push({ text: s, source, blockId: bid });- }- }-- function walkBlocks(tokens: marked.Token[]) {- for (const token of tokens) {- switch (token.type) {- case "heading": {- const bid = blockId++;- const text = extractInlineText(- (token as marked.Tokens.Heading).tokens,- ).trim();- if (text) result.push({ text, source: "heading", blockId: bid });- break;- }- case "paragraph": {- const bid = blockId++;- pushSentences(- extractInlineText((token as marked.Tokens.Paragraph).tokens),- "text",- bid,- );- break;- }- case "list": {- for (const item of (token as marked.Tokens.List).items) {- walkBlocks(item.tokens);- }- break;- }- case "blockquote": {- walkBlocks((token as marked.Tokens.Blockquote).tokens);- break;- }- case "table": {- const bid = blockId++;- const table = token as marked.Tokens.Table;- const headerText = table.header- .map(cell => extractInlineText(cell.tokens).trim())- .filter(Boolean)- .join(" | ");- if (headerText)- result.push({- text: headerText,- source: "table",- blockId: bid,- isHeader: true,- });- for (const row of table.rows) {- const rowText = row- .map(cell => extractInlineText(cell.tokens).trim())- .filter(Boolean)- .join(" | ");- if (rowText)- result.push({ text: rowText, source: "table", blockId: bid });- }- break;- }- case "code": {- const bid = blockId++;- const codeToken = token as marked.Tokens.Code;- const lang = codeToken.lang || undefined;- const lines = codeToken.text.split("\n");- for (const line of lines) {- const trimmed = line.trim();- if (trimmed)- result.push({- text: trimmed,- source: "code",- blockId: bid,- lang,- });- }- break;- }- case "hr":- case "space":- case "html":- break;- default: {- const bid = blockId++;- if ("tokens" in token && Array.isArray(token.tokens)) {- walkBlocks(token.tokens);- } else if ("text" in token && typeof token.text === "string") {- pushSentences(token.text, "text", bid);- }- break;- }- }- }- }-- walkBlocks(tokens);- return result;-}--function assembleAnswer(sentences: Sentence[], indices: number[]): string {- const selectedIndices = new Set(- indices.filter(i => i >= 0 && i < sentences.length),- );-- const tableHeaderIncluded = new Set<number>();- for (const i of selectedIndices) {- const s = sentences[i];- if (- s.source === "table" &&- !s.isHeader &&- !tableHeaderIncluded.has(s.blockId)- ) {- const headerIdx = sentences.findIndex(- h => h.blockId === s.blockId && h.isHeader,- );- if (headerIdx !== -1) {- selectedIndices.add(headerIdx);- tableHeaderIncluded.add(s.blockId);- }- }- }-- const selected = [...selectedIndices]- .sort((a, b) => a - b)- .map(i => ({ index: i, ...sentences[i] }));-- const parts: string[] = [];- let codeRun: { texts: string[]; lang?: string } | null = null;-- function flushCode() {- if (!codeRun) return;- const fence = codeRun.lang ? "```" + codeRun.lang : "```";- parts.push(fence + "\n" + codeRun.texts.join("\n") + "\n```");- codeRun = null;- }-- let tableRows: string[] | null = null;-- function formatTableRow(text: string): string {- return "| " + text.split(" | ").join(" | ") + " |";- }-- function startTable(cur: (typeof selected)[number]) {- tableRows = [];- if (cur.isHeader) {- const cols = cur.text.split(" | ");- const escaped = cols.map(c => c.replace(/\*/g, "\\*"));- tableRows.push("| " + escaped.join(" | ") + " |");- tableRows.push("| " + cols.map(() => "---").join(" | ") + " |");- } else {- tableRows.push(formatTableRow(cur.text));- }- }-- function appendTableRow(cur: (typeof selected)[number]) {- if (!tableRows) {- startTable(cur);- return;- }- if (cur.isHeader) {- const cols = cur.text.split(" | ");- const escaped = cols.map(c => c.replace(/\*/g, "\\*"));- tableRows.unshift("| " + escaped.join(" | ") + " |");- tableRows.splice(1, 0, "| " + cols.map(() => "---").join(" | ") + " |");- } else {- tableRows.push(formatTableRow(cur.text));- }- }-- function flushTable() {- if (!tableRows) return;- parts.push(tableRows.join("\n"));- tableRows = null;- }-- for (let k = 0; k < selected.length; k++) {- const cur = selected[k];- const prev = k > 0 ? selected[k - 1] : null;- const samePrevBlock =- prev && prev.blockId === cur.blockId && prev.source === cur.source;-- if (cur.source === "code") {- flushTable();- if (samePrevBlock && codeRun) {- codeRun.texts.push(cur.text);- } else {- flushCode();- codeRun = { texts: [cur.text], lang: cur.lang };- }- continue;- }-- if (cur.source === "table") {- flushCode();- if (samePrevBlock && tableRows) {- appendTableRow(cur);- } else {- flushTable();- startTable(cur);- }- continue;- }-- flushCode();- flushTable();-- if (samePrevBlock) {- parts[parts.length - 1] += " " + cur.text;- } else {- parts.push(cur.text);- }- }-- flushCode();- flushTable();- return parts.join("\n\n");-}-const DIRECT_QUOTE_MODEL = {id: "accounts/thomas-bfc570/models/gpt-oss-20b-query-finetune-2026-04-15#accounts/thomas-bfc570/deployments/gpt-oss-20b-query-finetune-2026-04-24",provider: "fireworks" as const,diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.tsindex 82cbe6678..8e6b02a35 100644--- a/apps/api/src/search/highlight-model.test.ts+++ b/apps/api/src/search/highlight-model.test.ts@@ -38,18 +38,15 @@ describe("generateHighlightsBatch", () => {expect(fetchMock).not.toHaveBeenCalled();});- it("posts every page to /batch_highlight with the bearer token", async () => {+ it("posts each page's spans as `lines` to /batch_highlight with the bearer token", async () => {const fetchMock = mockFetchOnce({- results: [- { pruned_markdown: "Highlight A\n" },- { pruned_markdown: "Highlight B\n" },- ],+ results: [{ highlights: [] }, { highlights: [] }],});await generateHighlightsBatch([- { query: "q1", markdown: "page one" },- { query: "q2", markdown: "page two" },+ { query: "q1", lines: ["a one", "b one"] },+ { query: "q2", lines: ["a two"] },],{ logger },);@@ -62,53 +59,73 @@ describe("generateHighlightsBatch", () => {const sent = JSON.parse(init.body);expect(sent.requests).toHaveLength(2);- expect(sent.requests[0]).toMatchObject({ query: "q1", markdown: "page one" });+ expect(sent.requests[0]).toMatchObject({+ query: "q1",+ lines: ["a one", "b one"],+ });// Threshold + budget are always sent so the cutoff matches the trained format.expect(typeof sent.requests[0].threshold).toBe("number");expect(typeof sent.requests[0].max_highlight_chars).toBe("number");});- it("maps pruned_markdown back to each item, trimmed, aligned by index", async () => {+ it("returns the selected span indices per page, aligned by request order", async () => {mockFetchOnce({results: [- { pruned_markdown: "First page highlight\n" },- { pruned_markdown: "Second page highlight\nmore\n" },+ {+ highlights: [+ { index: 0, score: 0.9 },+ { index: 2, score: 0.5 },+ ],+ },+ { highlights: [{ index: 1, score: 0.7 }] },],});const out = await generateHighlightsBatch([- { query: "q1", markdown: "a" },- { query: "q2", markdown: "b" },+ { query: "q1", lines: ["l0", "l1", "l2"] },+ { query: "q2", lines: ["l0", "l1"] },+ ],+ { logger },+ );++ expect(out).toEqual([[0, 2], [1]]);+ });++ it("returns an empty array for a page with no selected spans", async () => {+ mockFetchOnce({ results: [{ highlights: [] }, {}] });++ const out = await generateHighlightsBatch(+ [+ { query: "q", lines: ["a"] },+ { query: "q", lines: ["b"] },],{ logger },);- expect(out).toEqual([- "First page highlight",- "Second page highlight\nmore",- ]);+ expect(out).toEqual([[], []]);});- it("returns null for empty / missing pruned_markdown entries", async () => {+ it("ignores malformed (non-integer / missing) indices", async () => {mockFetchOnce({results: [- { pruned_markdown: "" },- { pruned_markdown: " \n" },- {},+ {+ highlights: [+ { index: 1, score: 0.9 },+ { score: 0.8 }, // missing index+ { index: 2.5, score: 0.7 }, // non-integer+ { index: 3, score: 0.6 },+ ],+ },],});const out = await generateHighlightsBatch(- [- { query: "q", markdown: "a" },- { query: "q", markdown: "b" },- { query: "q", markdown: "c" },- ],+ [{ query: "q", lines: ["a", "b", "c", "d"] }],{ logger },);- expect(out).toEqual([null, null, null]);+ expect(out).toEqual([[1, 3]]);});it("falls back to nulls (one per item) when the service errors", async () => {@@ -116,8 +133,8 @@ describe("generateHighlightsBatch", () => {const out = await generateHighlightsBatch([- { query: "q", markdown: "a" },- { query: "q", markdown: "b" },+ { query: "q", lines: ["a"] },+ { query: "q", lines: ["b"] },],{ logger },);diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.tsindex e41a192db..9dd43cfaf 100644--- a/apps/api/src/search/highlight-model.ts+++ b/apps/api/src/search/highlight-model.ts@@ -1,58 +1,68 @@import type { Logger } from "winston";import { config } from "../config";-// Query Highlights model service: a query-finetuned span scorer (no LLM). It-// splits each page's markdown into line-like candidate spans β the same shape-// the model was trained on β scores every span against the query, and returns-// the selected spans concatenated in page order as `pruned_markdown`. The-// service keeps a resident GPU model, so we send every page through one-// `/batch_highlight` call instead of one request per page. Endpoint + token are+// Query Highlights model service: a query-finetuned span scorer (no LLM). It was+// trained on the line-like spans `parseMarkdownToSentences` produces, so callers+// pass those spans as explicit `lines`, the service scores each against the+// query, and returns the selected spans as `highlights[]` with their `index`+// back into the input `lines` array. Callers map those indices through+// `assembleAnswer` to rebuild structure (tables/code). We push every page+// through one `/batch_highlight` call (resident GPU model). Endpoint + token are// config-gated (HIGHLIGHT_MODEL_URL / HIGHLIGHT_MODEL_TOKEN); callers must// confirm they're set via highlightsEnvReady() before invoking.// Experimental score cutoff for keeping a span, tuned against the line-span// format the model was trained on.const HIGHLIGHT_THRESHOLD = 0.08;-// Character budget for the assembled snippet; the service keeps the-// highest-scoring spans until the budget is reached.+// Character budget per page; the service keeps the highest-scoring spans until+// the budget is reached (measured on raw span text).const MAX_HIGHLIGHT_CHARS = 800;const REQUEST_TIMEOUT_MS = 30000;interface HighlightItem {query: string;- markdown: string;+ // Candidate spans for the page, in document order β from+ // parseMarkdownToSentences(markdown).map(s => s.text).+ lines: string[];+}++interface HighlightSpan {+ index?: number;+ score?: number;}interface HighlightResult {- pruned_markdown?: string;+ highlights?: HighlightSpan[];}interface BatchHighlightResponse {results?: HighlightResult[];}-// The service returns the selected spans joined with "\n" (often with a trailing-// newline). Trim to a clean snippet; null when nothing was selected.-function prunedToSnippet(result: HighlightResult | undefined): string | null {- const pruned = result?.pruned_markdown;- if (typeof pruned !== "string") return null;- const trimmed = pruned.trim();- return trimmed === "" ? null : trimmed;+// Pull the selected span indices out of one page's result. The service already+// applied the threshold / char budget, so we just collect the kept indices.+function selectedIndices(result: HighlightResult | undefined): number[] {+ const highlights = result?.highlights ?? [];+ const out: number[] = [];+ for (const h of highlights) {+ if (typeof h.index === "number" && Number.isInteger(h.index)) {+ out.push(h.index);+ }+ }+ return out;}/**- * Generate query-relevant highlights for many pages in a single batch call to- * the query-highlights model. Each page's markdown is sent as-is; the service- * splits it into the line-like spans used in training, scores them against the- * query, and returns the selected spans joined in page order- * (`pruned_markdown`). Returns an array aligned to `items`: the highlight- * snippet for each page, or null when nothing clears the threshold (or the whole- * batch fails β highlights are best-effort, callers keep the provider snippet).+ * Score many pages' candidate spans in a single batch call to the+ * query-highlights model. Returns an array aligned to `items`: for each page,+ * the indices (into that page's `lines`) the model selected, or null when the+ * whole batch fails (so callers can keep the provider snippet). A page with no+ * spans clearing the threshold yields an empty array, not null.*/export async function generateHighlightsBatch(items: HighlightItem[],opts: { logger: Logger },-): Promise<(string | null)[]> {+): Promise<(number[] | null)[]> {if (items.length === 0) return [];const controller = new AbortController();@@ -68,7 +78,7 @@ export async function generateHighlightsBatch(body: JSON.stringify({requests: items.map(item => ({query: item.query,- markdown: item.markdown,+ lines: item.lines,threshold: HIGHLIGHT_THRESHOLD,max_highlight_chars: MAX_HIGHLIGHT_CHARS,})),@@ -85,15 +95,15 @@ export async function generateHighlightsBatch(const data = (await res.json()) as BatchHighlightResponse;const results = data.results ?? [];- const snippets = items.map((_, i) => prunedToSnippet(results[i]));+ const indices = items.map((_, i) => selectedIndices(results[i]));opts.logger.info("query highlights batch generated", {pages: items.length,- withHighlights: snippets.filter(Boolean).length,+ withHighlights: indices.filter(x => x.length > 0).length,elapsedMs: Date.now() - start,});- return snippets;+ return indices;} catch (error) {opts.logger.warn("query highlights batch failed", {error: error instanceof Error ? error.message : String(error),diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.tsindex 9a6eb0a9f..116f31974 100644--- a/apps/api/src/search/highlights.ts+++ b/apps/api/src/search/highlights.ts@@ -11,6 +11,10 @@ import { parseMarkdown } from "../lib/html-to-markdown";import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";import type { ScrapeOptions } from "../controllers/v2/types";import { generateHighlightsBatch } from "./highlight-model";+import {+ parseMarkdownToSentences,+ assembleAnswer,+} from "../lib/highlight-spans";import { config } from "../config";// How far back into the index we're willing to reach for highlight source text.@@ -122,9 +126,12 @@ async function getIndexedMarkdownForURL(/*** For each search result: look up the URL in our index (last 30 days), and if* present, replace the provider snippet with query-relevant highlights generated- * from the indexed content. Index lookups run in parallel, then every hit is- * scored in a single batch call to the query-highlights model. Mutates- * `response` in place. Results not in the index keep their original snippet.+ * from the indexed content. Index lookups run in parallel; each hit's markdown+ * is split into the same candidate spans the highlight model was trained on+ * (parseMarkdownToSentences), every hit is scored in a single batch call, and+ * the selected span indices are reassembled with assembleAnswer (structure-aware+ * β rebuilds tables/code). Mutates `response` in place. Results not in the index+ * keep their original snippet.*/export async function applySearchHighlights(response: SearchV2Response,@@ -160,25 +167,36 @@ export async function applySearchHighlights(return { attempted, indexHits: 0, replaced: 0 };}- // Look up indexed markdown for every URL in parallel, keeping only the hits.+ // Look up indexed markdown for every URL in parallel, then split each hit into+ // candidate spans. Keep the parsed sentences around so we can reassemble the+ // model's selected indices back into structured markdown.const markdowns = await Promise.all(targets.map(t => getIndexedMarkdownForURL(t.url, logger)),);- const hits: { apply: (h: string) => void; markdown: string }[] = [];+ const hits: {+ apply: (h: string) => void;+ sentences: ReturnType<typeof parseMarkdownToSentences>;+ }[] = [];markdowns.forEach((markdown, i) => {- if (markdown) hits.push({ apply: targets[i].apply, markdown });+ if (!markdown) return;+ const sentences = parseMarkdownToSentences(markdown);+ if (sentences.length === 0) return;+ hits.push({ apply: targets[i].apply, sentences });});const indexHits = hits.length;- // Score every hit in one batch call, then apply the highlights that came back.+ // Score every hit in one batch call (candidate spans as explicit `lines`),+ // then reassemble each page's selected span indices into a snippet.let replaced = 0;if (indexHits > 0) {- const snippets = await generateHighlightsBatch(- hits.map(h => ({ query, markdown: h.markdown })),+ const perPageIndices = await generateHighlightsBatch(+ hits.map(h => ({ query, lines: h.sentences.map(s => s.text) })),{ logger },);- snippets.forEach((snippet, i) => {- if (snippet && snippet.trim() !== "") {+ perPageIndices.forEach((indices, i) => {+ if (!indices || indices.length === 0) return;+ const snippet = assembleAnswer(hits[i].sentences, indices);+ if (snippet.trim() !== "") {hits[i].apply(snippet);replaced++;}