Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
chore(api): experimental (WIP) (#3776)
Gergő Móricz8c6de40e336d
diff --git a/apps/api/src/search/highlight-model.ts b/apps/api/src/search/highlight-model.tsnew file mode 100644index 000000000..817c37f03--- /dev/null+++ b/apps/api/src/search/highlight-model.ts@@ -0,0 +1,203 @@+import type { Logger } from "winston";+import { config } from "../config";++// Semantic highlight model: it scores each sentence of a context against a+// question (semantic similarity), no LLM. Inference is fast (~10ms), but the+// model caps context at ~4k tokens, so we chunk the page, score every chunk,+// pool the per-sentence scores globally, and keep the best sentences. Endpoint+// is config-gated (HIGHLIGHT_MODEL_URL); callers must confirm it's set via+// highlightsEnvReady() before invoking.++// ~4 chars/token; 10k chars (~2.5k tokens) leaves headroom under the 4k cap for+// the question + tokenizer variance (markdown/code tokenizes denser than prose).+const CHUNK_CHARS = 10000;+// Bound load for very long pages (chunks run concurrently per result).+const MAX_CHUNKS = 10;+// Keep sentences scoring at/above this (matches the model's default threshold).+const SELECT_THRESHOLD = 0.5;+// Cap the assembled snippet length.+const MAX_SELECTED = 12;+const REQUEST_TIMEOUT_MS = 30000;++interface ScoredSentence {+ text: string;+ score: number;+ order: number; // global document order+}++// Strip markdown syntax so a highlight reads as plain prose: links/images keep+// their text, bare URLs and emphasis/heading/code marks go, line-number gutters+// from rendered code blocks are dropped, whitespace collapses.+function cleanSentence(text: string): string {+ return text+ .replace(/!\[[^\]]*\]\([^)]*\)/g, " ") // images+ .replace(/\[([^\]]*)\]\([^)]*\)/g, "$1") // links -> link text+ .replace(/https?:\/\/\S+/g, " ") // bare URLs+ .replace(/^\s*\d+\s*/, "") // leading code-block line number+ .replace(/[#*`>_~|\\]+/g, " ") // markdown marks+ .replace(/\s+/g, " ")+ .trim();+}++// Drop markdown/code artifacts (line numbers, "=====", "//", "[ 01 / 06 ]")+// that the semantic model sometimes scores highly. Real sentences have prose.+function isContentful(text: string): boolean {+ const letters = (text.match(/[a-zA-Z]/g) ?? []).length;+ return letters >= 15;+}++interface HighlightModelResponse {+ kept_sentences?: string[];+ sentence_probabilities?: number[];+}++/**+ * Split markdown into chunks under the model's token budget. Splits on blank+ * lines (paragraph boundaries) to avoid cutting mid-sentence; oversized+ * paragraphs are hard-split as a last resort. Capped at MAX_CHUNKS.+ */+function chunkMarkdown(markdown: string): string[] {+ const chunks: string[] = [];+ let current = "";++ const flush = () => {+ if (current.trim() !== "") chunks.push(current);+ current = "";+ };++ for (const para of markdown.split(/\n{2,}/)) {+ if (chunks.length >= MAX_CHUNKS) break;++ if (para.length > CHUNK_CHARS) {+ flush();+ for (+ let i = 0;+ i < para.length && chunks.length < MAX_CHUNKS;+ i += CHUNK_CHARS+ ) {+ chunks.push(para.slice(i, i + CHUNK_CHARS));+ }+ continue;+ }++ if (current.length + para.length + 2 > CHUNK_CHARS) {+ flush();+ }+ current += (current === "" ? "" : "\n\n") + para;+ }++ if (chunks.length < MAX_CHUNKS) flush();+ return chunks.slice(0, MAX_CHUNKS);+}++async function scoreChunk(+ question: string,+ context: string,+ logger: Logger,+): Promise<{ text: string; score: number }[]> {+ const controller = new AbortController();+ const timer = setTimeout(() => controller.abort(), REQUEST_TIMEOUT_MS);+ try {+ const res = await fetch(`${config.HIGHLIGHT_MODEL_URL}/v1/highlight`, {+ method: "POST",+ headers: { "Content-Type": "application/json" },+ body: JSON.stringify({+ question,+ context,+ // threshold 0 => every sentence is returned with its score, so we can+ // pool scores across chunks and select globally.+ threshold: 0.0,+ language: "auto",+ return_sentence_metrics: true,+ }),+ signal: controller.signal,+ });++ if (!res.ok) {+ const body = await res.text().catch(() => "");+ throw new Error(+ `highlight model HTTP ${res.status}: ${body.slice(0, 200)}`,+ );+ }++ const data = (await res.json()) as HighlightModelResponse;+ const sentences = data.kept_sentences ?? [];+ const probs = data.sentence_probabilities ?? [];+ const n = Math.min(sentences.length, probs.length);+ const out: { text: string; score: number }[] = [];+ for (let i = 0; i < n; i++) {+ out.push({ text: sentences[i], score: probs[i] });+ }+ return out;+ } finally {+ clearTimeout(timer);+ }+}++/**+ * Generate query-relevant highlights from page markdown using the semantic+ * highlight model. Returns the assembled highlight string (best sentences in+ * document order), or null if nothing clears the threshold / all chunks fail.+ */+export async function generateSemanticHighlights(+ markdown: string,+ query: string,+ opts: { logger: Logger },+): Promise<string | null> {+ const chunks = chunkMarkdown(markdown);+ if (chunks.length === 0) return null;++ const start = Date.now();+ const perChunk = await Promise.all(+ chunks.map(async (chunk, idx) => {+ try {+ return await scoreChunk(query, chunk, opts.logger);+ } catch (error) {+ opts.logger.warn("highlight model chunk failed", {+ error: error instanceof Error ? error.message : String(error),+ chunkIdx: idx,+ });+ return [];+ }+ }),+ );++ // Pool every scored sentence, preserving document order (Promise.all keeps+ // chunk order, and sentences within a chunk are already in order).+ const all: ScoredSentence[] = [];+ let order = 0;+ for (const chunkSentences of perChunk) {+ for (const s of chunkSentences) {+ const idx = order++;+ const cleaned = cleanSentence(s.text);+ if (isContentful(cleaned)) {+ all.push({ text: cleaned, score: s.score, order: idx });+ }+ }+ }+ if (all.length === 0) return null;++ // Pick the best: keep sentences at/above the threshold, cap to the top+ // MAX_SELECTED by score, then restore document order for readability.+ let selected = all.filter(s => s.score >= SELECT_THRESHOLD);+ if (selected.length === 0) return null;+ selected.sort((a, b) => b.score - a.score);+ selected = selected.slice(0, MAX_SELECTED);+ selected.sort((a, b) => a.order - b.order);++ const text = selected+ .map(s => s.text.trim())+ .join(" ")+ .replace(/\s+/g, " ")+ .trim();++ opts.logger.info("semantic highlights generated", {+ chunks: chunks.length,+ scoredSentences: all.length,+ selected: selected.length,+ topScore: all.reduce((m, s) => Math.max(m, s.score), 0),+ elapsedMs: Date.now() - start,+ });++ return text === "" ? null : text;+}diff --git a/apps/api/src/search/highlights.ts b/apps/api/src/search/highlights.tsnew file mode 100644index 000000000..116f5ff0d--- /dev/null+++ b/apps/api/src/search/highlights.ts@@ -0,0 +1,187 @@+import type { Logger } from "winston";+import { SearchV2Response } from "../lib/entities";+import {+ normalizeURLForIndex,+ hashURL,+ getIndexFromGCS,+ useIndex,+} from "../services";+import { indexGetRecent5 } from "../db/rpc";+import { parseMarkdown } from "../lib/html-to-markdown";+import { htmlTransform } from "../scraper/scrapeURL/lib/removeUnwantedElements";+import type { ScrapeOptions } from "../controllers/v2/types";+import { generateSemanticHighlights } from "./highlight-model";+import { config } from "../config";++// How far back into the index we're willing to reach for highlight source text.+const HIGHLIGHTS_INDEX_MAX_AGE_MS = 30 * 24 * 60 * 60 * 1000; // 30 days++/**+ * Whether the deployment has every dependency the highlights beta needs: the+ * index DB (to find cached content), the GCS index bucket (to fetch it), and the+ * highlight model endpoint (to score it). Missing any => silently skip.+ */+export function highlightsEnvReady(): boolean {+ return (+ useIndex && !!config.GCS_INDEX_BUCKET_NAME && !!config.HIGHLIGHT_MODEL_URL+ );+}++// Mirrors scrapeURLWithIndex: prefer the newest 2xx entry unless it sits behind+// this many more-recent error entries, in which case we surface the newest one.+const ERROR_COUNT_TO_REGISTER = 3;++// This whole module runs out-of-line from scrapeURL on purpose: it reads+// already-indexed content directly from the index DB + GCS instead of routing+// through the scrape engine. That keeps highlight generation off the critical+// scrape path and lets us experiment with latency freely.++/**+ * Fetch the most recent indexed markdown for a URL within the last 30 days.+ * Returns null when the URL isn't in the index (or the lookup fails) so callers+ * can fall back to the provider snippet.+ */+async function getIndexedMarkdownForURL(+ url: string,+ logger: Logger,+): Promise<string | null> {+ if (!useIndex) {+ return null;+ }++ try {+ const normalizedURL = normalizeURLForIndex(url);+ const urlHash = hashURL(normalizedURL);++ // Match the most common index variant (default scrape options) to maximize+ // hit rate: desktop, ads blocked, no screenshot, no location, no stealth.+ const rows = await indexGetRecent5({+ url_hash: urlHash,+ max_age_ms: HIGHLIGHTS_INDEX_MAX_AGE_MS,+ is_mobile: false,+ block_ads: true,+ feature_screenshot: false,+ feature_screenshot_fullscreen: false,+ location_country: null,+ location_languages: null,+ wait_time_ms: 0,+ is_stealth: false,+ min_age_ms: null,+ });++ if (!rows || rows.length === 0) {+ return null;+ }++ const newest200Index = rows.findIndex(+ x => x.status >= 200 && x.status < 300,+ );+ const selected =+ newest200Index >= ERROR_COUNT_TO_REGISTER || newest200Index === -1+ ? rows[0]+ : rows[newest200Index];++ const doc = await getIndexFromGCS(+ selected.id + ".json",+ logger.child({ module: "search/highlights", method: "getIndexFromGCS" }),+ );+ if (!doc || !doc.html) {+ return null;+ }++ // Skip raw base64 PDFs — they aren't useful as highlight source text.+ if (typeof doc.html === "string" && doc.html.startsWith("JVBERi")) {+ return null;+ }++ // The index stores rawHtml, so we must run the same cleaning the scrape+ // pipeline does (strip <style>/<script>/nav, extract main content) before+ // converting to markdown — otherwise CSS/JS leaks in and pollutes the+ // highlight source text.+ const cleanedHtml = await htmlTransform(doc.html, url, {+ onlyMainContent: true,+ includeTags: [],+ excludeTags: [],+ } as unknown as ScrapeOptions);++ const markdown = await parseMarkdown(cleanedHtml, { logger });+ return markdown && markdown.trim() !== "" ? markdown : null;+ } catch (error) {+ logger.warn("highlights: index lookup failed", {+ error: error instanceof Error ? error.message : String(error),+ url,+ });+ return null;+ }+}++async function highlightOne(+ url: string,+ query: string,+ logger: Logger,+ apply: (highlights: string) => void,+ counters: { indexHits: number; replaced: number },+): Promise<void> {+ const markdown = await getIndexedMarkdownForURL(url, logger);+ if (!markdown) {+ return;+ }+ counters.indexHits++;++ const highlights = await generateSemanticHighlights(markdown, query, {+ logger,+ });+ if (highlights && highlights.trim() !== "") {+ apply(highlights);+ counters.replaced++;+ }+}++/**+ * For each search result, in parallel: look up the URL in our index (last 30+ * days), and if present, replace the provider snippet with query-relevant+ * highlights generated from the indexed content. Mutates `response` in place.+ * Results not in the index keep their original snippet.+ */+export async function applySearchHighlights(+ response: SearchV2Response,+ query: string,+ logger: Logger,+): Promise<{ attempted: number; indexHits: number; replaced: number }> {+ const counters = { indexHits: 0, replaced: 0 };+ const tasks: Promise<void>[] = [];+ const start = Date.now();++ // Web results carry the snippet in `description` — replace it in place.+ for (const result of response.web ?? []) {+ if (!result.url) continue;+ tasks.push(+ highlightOne(result.url, query, logger, h => {+ result.description = h;+ }, counters),+ );+ }++ // News results carry the snippet in `snippet` — replace it in place.+ for (const result of response.news ?? []) {+ if (!result.url) continue;+ const url = result.url;+ tasks.push(+ highlightOne(url, query, logger, h => {+ result.snippet = h;+ }, counters),+ );+ }++ const attempted = tasks.length;+ await Promise.all(tasks);++ logger.info("Search highlights applied", {+ attempted,+ indexHits: counters.indexHits,+ replaced: counters.replaced,+ timeTakenMs: Date.now() - start,+ });++ return { attempted, indexHits: counters.indexHits, replaced: counters.replaced };+}