OpenRAG is a comprehensive, single package Retrieval-Augmented Generation platform built on Langflow, Docling, and Opensearch.
Python
/**
* One-line "why does this file match?" previews for search results.
*
* A `keyword` preview is an OpenSearch highlight fragment (matched terms wrapped
* in `<mark>`), trimmed to one line with `…` where text was cut away. A
* `semantic` preview is the sentence of the best chunk that shares the most
* words with the query, shown when nothing matched literally.
*/
import type { ChunkResult, File } from "@/app/api/queries/useGetSearchQuery";
import { getKnowledgeFileIdentity } from "@/lib/knowledge-table-state";
export const PREVIEW_MAX_CHARS = 180;
/** How many of the most relevant files get a preview line in the table. */
const TABLE_PREVIEW_FILE_COUNT = 3;
export interface SearchPreview {
/** One line; `keyword` previews wrap matched terms in `<mark>`. */
text: string;
kind: "keyword" | "semantic";
}
const MARK_LENGTH = "<mark></mark>".length;
const MARK_SPLIT = /(<mark>.*?<\/mark>)/;
export function stripMarks(fragment: string): string {
return fragment.replace(/<\/?mark>/g, "");
}
function isMarkPart(part: string): boolean {
return part.startsWith("<mark>") && part.endsWith("</mark>");
}
/** Keep at most `limit` visible characters without splitting a `<mark>`. */
function takePlain(source: string, limit: number): string {
let out = "";
let used = 0;
for (const part of source.split(MARK_SPLIT)) {
if (!part) continue;
const plainLength = isMarkPart(part)
? part.length - MARK_LENGTH
: part.length;
if (used + plainLength <= limit) {
out += part;
used += plainLength;
continue;
}
if (!isMarkPart(part)) out += part.slice(0, Math.max(limit - used, 0));
break;
}
return out;
}
function truncatePlain(text: string, limit: number): string {
if (text.length <= limit) return text;
const cut = text
.slice(0, limit)
.replace(/\s+\S*$/, "")
.replace(/[\s,;:-]+$/, "");
return `${cut || text.slice(0, limit)}…`;
}
/** Shrink an over-long fragment around its first `<mark>`, ending with `…`. */
function clipFragment(fragment: string): string {
const flat = fragment.replace(/\s+/g, " ").trim();
if (stripMarks(flat).length <= PREVIEW_MAX_CHARS) return flat;
const first = /<mark>.*?<\/mark>/.exec(flat);
if (!first) return truncatePlain(flat, PREVIEW_MAX_CHARS);
const before = flat.slice(0, first.index);
const match = first[0];
const after = flat.slice(first.index + match.length);
const budget = Math.max(PREVIEW_MAX_CHARS - (match.length - MARK_LENGTH), 0);
const keepBefore = Math.floor(budget / 3);
const keepAfter = budget - Math.min(keepBefore, before.length);
const clippedBefore =
before.length > keepBefore
? `…${before.slice(before.length - keepBefore).replace(/^\S*\s+/, "")}`
: before;
const kept = takePlain(after, keepAfter);
const wasCut = stripMarks(kept).length < stripMarks(after).length;
const clippedAfter = wasCut
? `${kept.replace(/\s+[^\s<>]*$/, "").replace(/[\s,;:-]+$/, "")}…`
: kept;
return `${clippedBefore}${match}${clippedAfter}`;
}
/** The sentence sharing the most words with the query (first one on ties). */
const CJK_REGEX =
/[\u3040-\u30ff\u3400-\u4dbf\u4e00-\u9fff\uf900-\ufaff\uac00-\ud7af]/u;
export function bestSentence(text: string, query: string): string | null {
const cleaned = text.trim();
if (!cleaned) return null;
const sentences = cleaned
.split(/(?<=[.!?])\s+|(?<=[。!?])\s*|\n{2,}/)
.map((sentence) => sentence.trim())
.filter(
(sentence) =>
sentence.length >= 20 ||
(CJK_REGEX.test(sentence) && sentence.length >= 8),
);
const candidates = sentences.length > 0 ? sentences : [cleaned];
const words = (value: string) =>
value.toLowerCase().match(/[\p{L}\p{N}_]+/gu) ?? [];
const queryWords = new Set(words(query));
let best = candidates[0];
let bestScore = -1;
for (const sentence of candidates) {
const sLower = sentence.toLowerCase();
const sentenceWords = new Set(words(sentence));
let score = 0;
for (const queryWord of queryWords) {
if (
sentenceWords.has(queryWord) ||
(CJK_REGEX.test(queryWord) && sLower.includes(queryWord))
) {
score++;
}
}
if (score > bestScore) {
best = sentence;
bestScore = score;
}
}
return truncatePlain(best.replace(/\s+/g, " "), PREVIEW_MAX_CHARS);
}
function previewFromChunk(chunk: ChunkResult): SearchPreview | null {
const fragment = chunk.highlights?.find((item) => item.trim().length > 0);
if (!fragment) return null;
let line = clipFragment(fragment);
const plain = stripMarks(fragment);
const start = chunk.text ? chunk.text.indexOf(plain) : -1;
if (start > 0 && !line.startsWith("…")) line = `…${line}`;
if (
start >= 0 &&
start + plain.length < chunk.text.length &&
!line.endsWith("…")
) {
line = `${line}…`;
}
return { text: line, kind: "keyword" };
}
function bestChunkScore(file: File): number {
return Math.max(...(file.chunks ?? []).map((chunk) => chunk.score), 0);
}
/** Preview for one file: a literal match wins, else the best semantic sentence. */
export function buildFilePreview(
file: File,
query: string,
): SearchPreview | null {
const chunks = [...(file.chunks ?? [])].sort((a, b) => b.score - a.score);
for (const chunk of chunks) {
const preview = previewFromChunk(chunk);
if (preview) return preview;
}
const top = chunks.find((chunk) => chunk.text?.trim());
const sentence = top ? bestSentence(top.text, query) : null;
return sentence ? { text: sentence, kind: "semantic" } : null;
}
/**
* Previews for the `count` most relevant files, keyed by file identity.
* Relevance is the best chunk score, so a file with one excellent chunk
* outranks one with many mediocre chunks.
*/
export function computeTopPreviews(
files: File[],
query: string,
count = TABLE_PREVIEW_FILE_COUNT,
): Map<string, SearchPreview> {
const previews = new Map<string, SearchPreview>();
if (!query.trim() || query.trim() === "*") return previews;
const ranked = files
.filter((file) => (file.chunks?.length ?? 0) > 0)
.sort((a, b) => bestChunkScore(b) - bestChunkScore(a))
.slice(0, count);
for (const file of ranked) {
const preview = buildFilePreview(file, query);
if (preview) previews.set(getKnowledgeFileIdentity(file), preview);
}
return previews;
}
/** Split `text` around the first case-insensitive occurrence of `query`. */
export function splitOnQuery(
text: string,
query: string,
): { before: string; match: string; after: string } {
const needle = query.trim().toLowerCase();
const index = needle ? text.toLowerCase().indexOf(needle) : -1;
if (index < 0) return { before: text, match: "", after: "" };
return {
before: text.slice(0, index),
match: text.slice(index, index + needle.length),
after: text.slice(index + needle.length),
};
}