Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

chore(api): experimental (WIP) (#3776)

Gergő Móricz8c6de40e336d
diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.ts
new file mode 100644
index 000000000..817c37f03
--- /dev/null
+++ b/apps/api/src/search/highlight-model.ts
@@ -0,0 +1,203 @@
+import type { Logger } from "winston";
+import { config } from "../config";
+
+// Semantic highlight model: it scores each sentence of a context against a
+// question (semantic similarity), no LLM. Inference is fast (~10ms), but the
+// model caps context at ~4k tokens, so we chunk the page, score every chunk,
+// pool the per-sentence scores globally, and keep the best sentences. Endpoint
+// is config-gated (HIGHLIGHT_MODEL_URL); callers must confirm it's set via
+// highlightsEnvReady() before invoking.
+
+// ~4 chars/token; 10k chars (~2.5k tokens) leaves headroom under the 4k cap for
+// the question + tokenizer variance (markdown/code tokenizes denser than prose).
+const CHUNK_CHARS = 10000;
+// Bound load for very long pages (chunks run concurrently per result).
+const MAX_CHUNKS = 10;
+// Keep sentences scoring at/above this (matches the model's default threshold).
+const SELECT_THRESHOLD = 0.5;
+// Cap the assembled snippet length.
+const MAX_SELECTED = 12;
+const REQUEST_TIMEOUT_MS = 30000;
+
+interface ScoredSentence {
+ text: string;
+ score: number;
+ order: number; // global document order
+}
+
+// Strip markdown syntax so a highlight reads as plain prose: links/images keep
+// their text, bare URLs and emphasis/heading/code marks go, line-number gutters
+// from rendered code blocks are dropped, whitespace collapses.
+function cleanSentence(text: string): string {
+ return text
+ .replace(/!\[[^\]]*\]\([^)]*\)/g, " ") // images
+ .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") // links -> link text
+ .replace(/https?:\/\/\S+/g, " ") // bare URLs
+ .replace(/^\s*\d+\s*/, "") // leading code-block line number
+ .replace(/[#*`>_~|\\]+/g, " ") // markdown marks
+ .replace(/\s+/g, " ")
+ .trim();
+}
+
+// Drop markdown/code artifacts (line numbers, "=====", "//", "[ 01 / 06 ]")
+// that the semantic model sometimes scores highly. Real sentences have prose.
+function isContentful(text: string): boolean {
+ const letters = (text.match(/[a-zA-Z]/g) ?? []).length;
+ return letters >= 15;
+}
+
+interface HighlightModelResponse {
+ kept_sentences?: string[];
+ sentence_probabilities?: number[];
+}
+
+/**
+ * Split markdown into chunks under the model's token budget. Splits on blank
+ * lines (paragraph boundaries) to avoid cutting mid-sentence; oversized
+ * paragraphs are hard-split as a last resort. Capped at MAX_CHUNKS.
+ */
+function chunkMarkdown(markdown: string): string[] {
+ const chunks: string[] = [];
+ let current = "";
+
+ const flush = () => {
+ if (current.trim() !== "") chunks.push(current);
+ current = "";
+ };
+
+ for (const para of markdown.split(/\n{2,}/)) {
+ if (chunks.length >= MAX_CHUNKS) break;
+
+ if (para.length > CHUNK_CHARS) {
+ flush();
+ for (
+ let i = 0;
+ i < para.length && chunks.length < MAX_CHUNKS;
+ i += CHUNK_CHARS
+ ) {
+ chunks.push(para.slice(i, i + CHUNK_CHARS));
+ }
+ continue;
+ }
+
+ if (current.length + para.length + 2 > CHUNK_CHARS) {
+ flush();
+ }
+ current += (current === "" ? "" : "\n\n") + para;
+ }
+
+ if (chunks.length < MAX_CHUNKS) flush();
+ return chunks.slice(0, MAX_CHUNKS);
+}
+
+async function scoreChunk(
+ question: string,
+ context: string,
+ logger: Logger,
+): Promise<{ text: string; score: number }[]> {
+ const controller = new AbortController();
+ const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);
+ try {
+ const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/v1/highlight`, {
+ method: "POST",
+ headers: { "Content-Type": "application/json" },
+ body: JSON.stringify({
+ question,
+ context,
+ // threshold 0 => every sentence is returned with its score, so we can
+ // pool scores across chunks and select globally.
+ threshold: 0.0,
+ language: "auto",
+ return_sentence_metrics: true,
+ }),
+ signal: controller.signal,
+ });
+
+ if (!res.ok) {
+ const body = await res.text().catch(() => "");
+ throw new Error(
+ `highlight model HTTP ${res.status}: ${body.slice(0, 200)}`,
+ );
+ }
+
+ const data = (await res.json()) as HighlightModelResponse;
+ const sentences = data.kept_sentences ?? [];
+ const probs = data.sentence_probabilities ?? [];
+ const n = Math.min(sentences.length, probs.length);
+ const out: { text: string; score: number }[] = [];
+ for (let i = 0; i < n; i++) {
+ out.push({ text: sentences[i], score: probs[i] });
+ }
+ return out;
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
+/**
+ * Generate query-relevant highlights from page markdown using the semantic
+ * highlight model. Returns the assembled highlight string (best sentences in
+ * document order), or null if nothing clears the threshold / all chunks fail.
+ */
+export async function generateSemanticHighlights(
+ markdown: string,
+ query: string,
+ opts: { logger: Logger },
+): Promise<string | null> {
+ const chunks = chunkMarkdown(markdown);
+ if (chunks.length === 0) return null;
+
+ const start = Date.now();
+ const perChunk = await Promise.all(
+ chunks.map(async (chunk, idx) => {
+ try {
+ return await scoreChunk(query, chunk, opts.logger);
+ } catch (error) {
+ opts.logger.warn("highlight model chunk failed", {
+ error: error instanceof Error ? error.message : String(error),
+ chunkIdx: idx,
+ });
+ return [];
+ }
+ }),
+ );
+
+ // Pool every scored sentence, preserving document order (Promise.all keeps
+ // chunk order, and sentences within a chunk are already in order).
+ const all: ScoredSentence[] = [];
+ let order = 0;
+ for (const chunkSentences of perChunk) {
+ for (const s of chunkSentences) {
+ const idx = order++;
+ const cleaned = cleanSentence(s.text);
+ if (isContentful(cleaned)) {
+ all.push({ text: cleaned, score: s.score, order: idx });
+ }
+ }
+ }
+ if (all.length === 0) return null;
+
+ // Pick the best: keep sentences at/above the threshold, cap to the top
+ // MAX_SELECTED by score, then restore document order for readability.
+ let selected = all.filter(s => s.score >= SELECT_THRESHOLD);
+ if (selected.length === 0) return null;
+ selected.sort((a, b) => b.score - a.score);
+ selected = selected.slice(0, MAX_SELECTED);
+ selected.sort((a, b) => a.order - b.order);
+
+ const text = selected
+ .map(s => s.text.trim())
+ .join(" ")
+ .replace(/\s+/g, " ")
+ .trim();
+
+ opts.logger.info("semantic highlights generated", {
+ chunks: chunks.length,
+ scoredSentences: all.length,
+ selected: selected.length,
+ topScore: all.reduce((m, s) => Math.max(m, s.score), 0),
+ elapsedMs: Date.now() - start,
+ });
+
+ return text === "" ? null : text;
+}
diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.ts
new file mode 100644
index 000000000..116f5ff0d
--- /dev/null
+++ b/apps/api/src/search/highlights.ts
@@ -0,0 +1,187 @@
+import type { Logger } from "winston";
+import { SearchV2Response } from "../lib/entities";
+import {
+ normalizeURLForIndex,
+ hashURL,
+ getIndexFromGCS,
+ useIndex,
+} from "../services";
+import { indexGetRecent5 } from "../db/rpc";
+import { parseMarkdown } from "../lib/html-to-markdown";
+import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";
+import type { ScrapeOptions } from "../controllers/v2/types";
+import { generateSemanticHighlights } from "./highlight-model";
+import { config } from "../config";
+
+// How far back into the index we're willing to reach for highlight source text.
+const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days
+
+/**
+ * Whether the deployment has every dependency the highlights beta needs: the
+ * index DB (to find cached content), the GCS index bucket (to fetch it), and the
+ * highlight model endpoint (to score it). Missing any => silently skip.
+ */
+export function highlightsEnvReady(): boolean {
+ return (
+ useIndex && !!config.GCS_INDEX_BUCKET_NAME && !!config.HIGHLIGHT_MODEL_URL
+ );
+}
+
+// Mirrors scrapeURLWithIndex: prefer the newest 2xx entry unless it sits behind
+// this many more-recent error entries, in which case we surface the newest one.
+const ERROR_COUNT_TO_REGISTER = 3;
+
+// This whole module runs out-of-line from scrapeURL on purpose: it reads
+// already-indexed content directly from the index DB + GCS instead of routing
+// through the scrape engine. That keeps highlight generation off the critical
+// scrape path and lets us experiment with latency freely.
+
+/**
+ * Fetch the most recent indexed markdown for a URL within the last 30 days.
+ * Returns null when the URL isn't in the index (or the lookup fails) so callers
+ * can fall back to the provider snippet.
+ */
+async function getIndexedMarkdownForURL(
+ url: string,
+ logger: Logger,
+): Promise<string | null> {
+ if (!useIndex) {
+ return null;
+ }
+
+ try {
+ const normalizedURL = normalizeURLForIndex(url);
+ const urlHash = hashURL(normalizedURL);
+
+ // Match the most common index variant (default scrape options) to maximize
+ // hit rate: desktop, ads blocked, no screenshot, no location, no stealth.
+ const rows = await indexGetRecent5({
+ url_hash: urlHash,
+ max_age_ms: HIGHLIGHTS_INDEX_MAX_AGE_MS,
+ is_mobile: false,
+ block_ads: true,
+ feature_screenshot: false,
+ feature_screenshot_fullscreen: false,
+ location_country: null,
+ location_languages: null,
+ wait_time_ms: 0,
+ is_stealth: false,
+ min_age_ms: null,
+ });
+
+ if (!rows || rows.length === 0) {
+ return null;
+ }
+
+ const newest200Index = rows.findIndex(
+ x => x.status >= 200 && x.status < 300,
+ );
+ const selected =
+ newest200Index >= ERROR_COUNT_TO_REGISTER || newest200Index === -1
+ ? rows[0]
+ : rows[newest200Index];
+
+ const doc = await getIndexFromGCS(
+ selected.id + ".json",
+ logger.child({ module: "search/highlights", method: "getIndexFromGCS" }),
+ );
+ if (!doc || !doc.html) {
+ return null;
+ }
+
+ // Skip raw base64 PDFs — they aren't useful as highlight source text.
+ if (typeof doc.html === "string" && doc.html.startsWith("JVBERi")) {
+ return null;
+ }
+
+ // The index stores rawHtml, so we must run the same cleaning the scrape
+ // pipeline does (strip <style>/<script>/nav, extract main content) before
+ // converting to markdown — otherwise CSS/JS leaks in and pollutes the
+ // highlight source text.
+ const cleanedHtml = await htmlTransform(doc.html, url, {
+ onlyMainContent: true,
+ includeTags: [],
+ excludeTags: [],
+ } as unknown as ScrapeOptions);
+
+ const markdown = await parseMarkdown(cleanedHtml, { logger });
+ return markdown && markdown.trim() !== "" ? markdown : null;
+ } catch (error) {
+ logger.warn("highlights: index lookup failed", {
+ error: error instanceof Error ? error.message : String(error),
+ url,
+ });
+ return null;
+ }
+}
+
+async function highlightOne(
+ url: string,
+ query: string,
+ logger: Logger,
+ apply: (highlights: string) => void,
+ counters: { indexHits: number; replaced: number },
+): Promise<void> {
+ const markdown = await getIndexedMarkdownForURL(url, logger);
+ if (!markdown) {
+ return;
+ }
+ counters.indexHits++;
+
+ const highlights = await generateSemanticHighlights(markdown, query, {
+ logger,
+ });
+ if (highlights && highlights.trim() !== "") {
+ apply(highlights);
+ counters.replaced++;
+ }
+}
+
+/**
+ * For each search result, in parallel: look up the URL in our index (last 30
+ * days), and if present, replace the provider snippet with query-relevant
+ * highlights generated from the indexed content. Mutates `response` in place.
+ * Results not in the index keep their original snippet.
+ */
+export async function applySearchHighlights(
+ response: SearchV2Response,
+ query: string,
+ logger: Logger,
+): Promise<{ attempted: number; indexHits: number; replaced: number }> {
+ const counters = { indexHits: 0, replaced: 0 };
+ const tasks: Promise<void>[] = [];
+ const start = Date.now();
+
+ // Web results carry the snippet in `description` — replace it in place.
+ for (const result of response.web ?? []) {
+ if (!result.url) continue;
+ tasks.push(
+ highlightOne(result.url, query, logger, h => {
+ result.description = h;
+ }, counters),
+ );
+ }
+
+ // News results carry the snippet in `snippet` — replace it in place.
+ for (const result of response.news ?? []) {
+ if (!result.url) continue;
+ const url = result.url;
+ tasks.push(
+ highlightOne(url, query, logger, h => {
+ result.snippet = h;
+ }, counters),
+ );
+ }
+
+ const attempted = tasks.length;
+ await Promise.all(tasks);
+
+ logger.info("Search highlights applied", {
+ attempted,
+ indexHits: counters.indexHits,
+ replaced: counters.replaced,
+ timeTakenMs: Date.now() - start,
+ });
+
+ return { attempted, indexHits: counters.indexHits, replaced: counters.replaced };
+}