Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/search): cap highlights at top_k=12 per page
mogerybec36dda1c35
diff --git a/apps/api/src/search/highlight-model.test.ts b/apps/api/src/search/highlight-model.test.tsindex 8e6b02a35..a363d437a 100644--- a/apps/api/src/search/highlight-model.test.ts+++ b/apps/api/src/search/highlight-model.test.ts@@ -63,8 +63,9 @@ describe("generateHighlightsBatch", () => {query: "q1",lines: ["a one", "b one"],});- // Threshold + budget are always sent so the cutoff matches the trained format.+ // Threshold + top_k + budget are always sent so the cutoff matches the trained format.expect(typeof sent.requests[0].threshold).toBe("number");+ expect(typeof sent.requests[0].top_k).toBe("number");expect(typeof sent.requests[0].max_highlight_chars).toBe("number");});diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.tsindex 9dd43cfaf..e7abfacc5 100644--- a/apps/api/src/search/highlight-model.ts+++ b/apps/api/src/search/highlight-model.ts@@ -14,6 +14,8 @@ import { config } from "../config";// Experimental score cutoff for keeping a span, tuned against the line-span// format the model was trained on.const HIGHLIGHT_THRESHOLD = 0.08;+// Cap on how many spans to keep per page after thresholding (highest-scoring).+const HIGHLIGHT_TOP_K = 12;// Character budget per page; the service keeps the highest-scoring spans until// the budget is reached (measured on raw span text).const MAX_HIGHLIGHT_CHARS = 800;@@ -80,6 +82,7 @@ export async function generateHighlightsBatch(query: item.query,lines: item.lines,threshold: HIGHLIGHT_THRESHOLD,+ top_k: HIGHLIGHT_TOP_K,max_highlight_chars: MAX_HIGHLIGHT_CHARS,})),}),