Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: add support for highlighting search results

Abdullah Atta66041a45f6e6
diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.ts
index d2c9d56eb..533677c26 100644
--- a/packages/core/src/api/lookup.ts
+++ b/packages/core/src/api/lookup.ts
@@ -20,7 +20,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.
import { match } from "fuzzyjs";
import Database from "./index.js";
import {
+ HighlightedResult,
Item,
+ Match,
Note,
Notebook,
Reminder,
@@ -36,7 +38,8 @@ import { rebuildSearchIndex } from "../database/fts.js";
import { transformQuery } from "../utils/query-transformer.js";
import { getSortSelectors, groupArray } from "../utils/grouping.js";
import { fuzzy } from "../utils/fuzzy.js";
-import { extractText } from "../utils/html-parser.js";
+import { extractMatchingBlocks, extractText } from "../utils/html-parser.js";
+import { findOrAdd } from "../utils/array.js";
type SearchResults<T> = {
sorted: (sortOptions?: SortOptions) => Promise<VirtualizedGrouping<T>>;
@@ -53,100 +56,170 @@ type FuzzySearchField<T> = {
export default class Lookup {
constructor(private readonly db: Database) {}
- notes(query: string, notes?: FilteredSelector<Note>): SearchResults<Note> {
- return this.toSearchResults(async (sortOptions) => {
- const db = this.db.sql() as unknown as Kysely<RawDatabaseSchema>;
- const excludedIds = this.db.trash.cache.notes;
-
- const { query: transformedQuery, tokens } = transformQuery(query);
-
- const resultsA: string[] =
- transformedQuery.length === 0
- ? []
- : await db
- .selectFrom((eb) =>
- eb
- .selectFrom("notes_fts")
- .$if(!!notes, (eb) =>
- eb.where("id", "in", notes!.filter.select("id"))
- )
- .$if(excludedIds.length > 0, (eb) =>
- eb.where("id", "not in", excludedIds)
- )
- .where("title", "match", transformedQuery)
- .select(["id", sql<number>`rank * 10`.as("rank")])
- .unionAll((eb) =>
- eb
- .selectFrom("content_fts")
- .$if(!!notes, (eb) =>
- eb.where("noteId", "in", notes!.filter.select("id"))
- )
- .$if(excludedIds.length > 0, (eb) =>
- eb.where("noteId", "not in", excludedIds)
- )
- .where("data", "match", transformedQuery)
- .select(["noteId as id", "rank"])
- .$castTo<{
- id: string;
- rank: number;
- }>()
- )
- .as("results")
- )
- .select(["results.id"])
- .groupBy("results.id")
- .orderBy(
- sql`SUM(results.rank)`,
- sortOptions?.sortDirection || "desc"
- )
- .execute()
- .catch((e) => {
- logger.error(e, `Error while searching`, { query });
- return [];
- })
- .then((r) => r.map((r) => r.id));
-
- const smallTokens = Array.from(
- new Set(
- tokens.filter((token) => token.length < 3 && token !== "OR")
- ).values()
- );
- if (smallTokens.length === 0) return resultsA;
+ async notes(
+ query: string,
+ notes?: FilteredSelector<Note>
+ ): Promise<VirtualizedGrouping<HighlightedResult>> {
+ const db = this.db.sql() as unknown as Kysely<RawDatabaseSchema>;
+ const excludedIds = this.db.trash.cache.notes;
- const results = [];
+ const { query: transformedQuery, tokens } = transformQuery(query);
- const titles = await db
- .selectFrom("notes")
- .$if(!!transformedQuery && resultsA.length > 0, (eb) =>
- eb.where("id", "in", resultsA)
+ let mergedResults: HighlightedResult[] = [];
+ if (transformedQuery.length > 0) {
+ const results = await db
+ .selectFrom((eb) =>
+ eb
+ .selectFrom("notes_fts")
+ .$if(!!notes, (eb) =>
+ eb.where("id", "in", notes!.filter.select("id"))
+ )
+ .$if(excludedIds.length > 0, (eb) =>
+ eb.where("id", "not in", excludedIds)
+ )
+ .where("title", "match", transformedQuery)
+ .select([
+ "id",
+ sql<string>`'title'`.as("type"),
+ sql<string>`highlight(notes_fts, 1, '<nnmark>', '</nnmark>')`.as(
+ "match"
+ ),
+ sql<number>`rank * 10`.as("rank")
+ ])
+ .unionAll((eb) =>
+ eb
+ .selectFrom("content_fts")
+ .$if(!!notes, (eb) =>
+ eb.where("noteId", "in", notes!.filter.select("id"))
+ )
+ .$if(excludedIds.length > 0, (eb) =>
+ eb.where("noteId", "not in", excludedIds)
+ )
+ .where("data", "match", transformedQuery)
+ .select([
+ "noteId as id",
+ sql<string>`'content'`.as("type"),
+ sql<string>`highlight(content_fts, 2, '<nnmark>', '</nnmark>')`.as(
+ "match"
+ ),
+ "rank"
+ ])
+ .$castTo<{
+ id: string;
+ type: string;
+ rank: number;
+ match: string;
+ }>()
+ )
+ .as("results")
)
- .select(["id", "title"])
- .execute();
+ .select(["results.id", "results.match", "results.type", "results.rank"])
+ .execute()
+ .catch((e) => {
+ logger.error(e, `Error while searching`, { query });
+ return [];
+ });
- const htmls = await db
- .selectFrom("content")
- .$if(!!transformedQuery && resultsA.length > 0, (eb) =>
- eb.where("noteId", "in", resultsA)
- )
- .select(["data", "noteId as id"])
- .$castTo<{ data: string; id: string }>()
- .execute();
-
- for (let i = 0; i < titles.length; i++) {
- const title = titles[i];
- const html = htmls.find((h) => h.id === title.id);
- const text = html ? extractText(html.data) : "";
-
- if (
- smallTokens.every((token) => !!title.title?.includes(token)) ||
- smallTokens.every((token) => !!text?.includes(token))
- ) {
- results.push(title.id);
+ for (const result of results) {
+ const old = findOrAdd(mergedResults, (r) => r.id === result.id, {
+ type: "searchResult",
+ id: result.id,
+ content: [],
+ title: []
+ });
+
+ if (result.type === "content")
+ old.content = extractMatchingBlocks(result.match, "nnmark").flatMap(
+ (block) => {
+ return splitHighlightedMatch(block);
+ }
+ );
+ if (result.type === "title")
+ old.title = splitHighlightedMatch(result.match).flatMap((m) => m);
+ }
+
+ const resultsWithMissingTitle = mergedResults
+ .filter((r) => !r.title.length)
+ .map((r) => r.id);
+
+ if (resultsWithMissingTitle.length > 0) {
+ const titles = await db
+ .selectFrom("notes")
+ .where("id", "in", resultsWithMissingTitle)
+ .select(["id", "title"])
+ .execute();
+ for (const title of titles) {
+ const result = mergedResults.find((r) => r.id === title.id);
+ if (!result || !title.title) continue;
+ result.title = stringToMatch(title.title);
}
}
- return results;
- }, notes || this.db.notes.all);
+ mergedResults = mergedResults.filter((r) => !!r.title);
+ }
+
+ const smallTokens = Array.from(
+ new Set(
+ tokens.filter((token) => token.length < 3 && token !== "OR")
+ ).values()
+ );
+
+ if (smallTokens.length === 0)
+ return arrayToVirtualizedGrouping(
+ mergedResults,
+ this.db.options.batchSize
+ );
+
+ const ids = mergedResults.map((r) => r.id);
+ const titles = await db
+ .selectFrom("notes")
+ .$if(!!transformedQuery && ids.length > 0, (eb) =>
+ eb.where("id", "in", ids)
+ )
+ .select(["id", "title"])
+ .execute();
+
+ const htmls = await db
+ .selectFrom("content")
+ .where("content.locked", "!=", true)
+ .$if(!!transformedQuery && ids.length > 0, (eb) =>
+ eb.where("noteId", "in", ids)
+ )
+ .select(["data", "noteId as id"])
+ .$castTo<{ data: string; id: string }>()
+ .execute();
+
+ for (let i = 0; i < titles.length; i++) {
+ const title = titles[i];
+ const result = findOrAdd(mergedResults, (r) => r.id === title.id, {
+ id: title.id,
+ title: stringToMatch(title.title || ""),
+ type: "searchResult",
+ content: []
+ });
+
+ const html = htmls.find((h) => h.id === title.id);
+ const text = html ? extractText(html.data) : "";
+
+ if (
+ title.title &&
+ smallTokens.every((token) => !!title.title?.includes(token))
+ ) {
+ result.title.push(
+ ...splitHighlightedMatch(
+ highlightQueries(title.title, smallTokens)
+ ).flatMap((m) => m)
+ );
+ }
+
+ if (text && smallTokens.every((token) => !!text?.includes(token))) {
+ result.content.push(
+ ...splitHighlightedMatch(highlightQueries(text, smallTokens))
+ );
+ }
+ }
+
+ return arrayToVirtualizedGrouping(mergedResults, this.db.options.batchSize);
}
notebooks(query: string) {
@@ -332,3 +405,219 @@ export default class Lookup {
await rebuildSearchIndex(db);
}
}
+
+function highlightQueries(text: string, queries: string[]): string {
+ if (!text || !queries.length) return text;
+
+ // Collect all ranges
+ const ranges = [];
+ const lowerText = text.toLowerCase();
+
+ for (let i = 0; i < queries.length; i++) {
+ const query = queries[i];
+ const lowerQuery = query.toLowerCase();
+ const queryLen = query.length;
+ let pos = 0;
+
+ while ((pos = lowerText.indexOf(lowerQuery, pos)) !== -1) {
+ ranges.push({
+ start: pos,
+ end: pos + queryLen,
+ len: queryLen
+ });
+ pos += 1;
+ }
+ }
+
+ if (!ranges.length) return text;
+
+ // Sort by start position, then by length (longer first)
+ ranges.sort((a, b) => a.start - b.start || b.len - a.len);
+
+ // Filter overlaps and merge adjacent ranges
+ const merged = [ranges[0]];
+ for (let i = 1; i < ranges.length; i++) {
+ const current = ranges[i];
+ const previous = merged[merged.length - 1];
+
+ if (current.start > previous.end) {
+ // No overlap or adjacency - add as new range
+ merged.push(current);
+ } else if (current.start === previous.end) {
+ // Adjacent ranges - merge them
+ previous.end = current.end;
+ previous.len = previous.end - previous.start;
+ }
+ // Overlapping ranges are skipped
+ }
+
+ // Build result using array of parts
+ const parts = [];
+ let lastEnd = 0;
+
+ for (const { start, end } of merged) {
+ if (start > lastEnd) {
+ parts.push(text.slice(lastEnd, start));
+ }
+ parts.push("<nnmark>", text.slice(start, end), "</nnmark>");
+ lastEnd = end;
+ }
+
+ if (lastEnd < text.length) {
+ parts.push(text.slice(lastEnd));
+ }
+
+ return parts.join("");
+}
+
+function arrayToVirtualizedGrouping<T extends { id: string }>(
+ array: T[],
+ batchSize: number
+): VirtualizedGrouping<T> {
+ return new VirtualizedGrouping<T>(
+ array.length,
+ batchSize,
+ () => Promise.resolve(array.map((c) => c.id)),
+ async (start, end) => {
+ const items = array.slice(start, end);
+ return {
+ ids: items.map((i) => i.id),
+ items
+ };
+ },
+ (items) => groupArray(items, () => `${items.length} results`)
+ );
+}
+
+function splitHighlightedMatch(text: string): Match[][] {
+ const parts = text.split(/<nnmark>(.*?)<\/nnmark>/g);
+ const allMatches: Match[][] = [];
+ let matches: Match[] = [];
+ let totalLength = 0;
+
+ for (let i = 0; i < parts.length - 1; i += 2) {
+ const prefix = parts[i];
+ const match = parts[i + 1];
+ // const suffix = parts[i + 2] || "";
+ const matchLength = prefix.length + match.length; // + suffix.length;
+
+ if (totalLength > 120 && matches.length > 0) {
+ matches[matches.length - 1].suffix += "...";
+ allMatches.push(matches);
+ matches = [];
+ totalLength = 0;
+ }
+
+ matches.push({
+ match,
+ prefix,
+ suffix: ""
+ });
+
+ totalLength += matchLength;
+ }
+
+ if (matches.length > 0) {
+ matches[matches.length - 1].suffix += parts[parts.length - 1];
+ allMatches.push(matches);
+ }
+
+ for (const matches of allMatches) {
+ const totalLength = matches.reduce(
+ (length, curr) =>
+ length + curr.match.length + curr.prefix.length + curr.suffix.length,
+ 0
+ );
+ if (totalLength > 200) {
+ const start = matches[0];
+ const end = matches[matches.length - 1];
+
+ const centered = centerMatch(
+ start.prefix,
+ end.suffix,
+ totalLength - (start.prefix.length + end.suffix.length),
+ {
+ maxLength: 200
+ }
+ );
+
+ start.prefix = centered.prefix || " ";
+ end.suffix = centered.suffix || " ";
+ }
+ }
+
+ return allMatches;
+}
+
+interface CenterOptions {
+ maxLength?: number; // Maximum total length of output
+ minContext?: number; // Minimum context on each side
+ ellipsis?: string; // String to use for truncation
+ preferLeft?: boolean; // Prefer more context on left when odd length
+}
+
+function centerMatch(
+ prefix: string,
+ suffix: string,
+ matchLength: number,
+ options: CenterOptions = {}
+): { prefix?: string; suffix?: string } {
+ const {
+ maxLength = 120,
+ minContext = 20,
+ ellipsis = "...",
+ preferLeft = true
+ } = options;
+
+ // Handle edge cases
+ if (!match) return {};
+ if (matchLength >= maxLength) return {};
+
+ // Calculate available space for context
+ const availableSpace = maxLength - matchLength;
+
+ // Calculate initial context lengths
+ let leftLength = Math.floor(availableSpace / 2);
+ let rightLength = availableSpace - leftLength;
+
+ // Adjust if we prefer left context
+ if (preferLeft && availableSpace % 2 !== 0) {
+ leftLength++;
+ rightLength--;
+ }
+
+ // Ensure minimum context if possible
+ if (leftLength < minContext && prefix.length > leftLength) {
+ const diff = Math.min(rightLength - minContext, minContext - leftLength);
+ if (diff > 0) {
+ leftLength += diff;
+ rightLength -= diff;
+ }
+ } else if (rightLength < minContext && suffix.length > rightLength) {
+ const diff = Math.min(leftLength - minContext, minContext - rightLength);
+ if (diff > 0) {
+ rightLength += diff;
+ leftLength -= diff;
+ }
+ }
+
+ // Build result
+ const left =
+ prefix.length > leftLength ? ellipsis + prefix.slice(-leftLength) : prefix;
+ const right =
+ suffix.length > rightLength
+ ? suffix.slice(0, rightLength) + ellipsis
+ : suffix;
+
+ return { prefix: left, suffix: right };
+}
+
+function stringToMatch(str: string): Match[] {
+ return [
+ {
+ prefix: str,
+ match: "",
+ suffix: ""
+ }
+ ];
+}
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
index e9f6cbe3f..2d8a5df2c 100644
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -488,6 +488,18 @@ export interface Vault extends BaseItem<"vault"> {
key: Cipher<"base64">;
}
+export type Match = {
+ prefix: string;
+ match: string;
+ suffix: string;
+};
+export interface HighlightedResult {
+ type: "searchResult";
+ id: string;
+ title: Match[];
+ content: Match[][];
+}
+
export interface DeletedItem {
id: string;
deleted: true;
diff --git a/packages/core/src/utils/array.ts b/packages/core/src/utils/array.ts
index d274322d0..8d92d52d0 100644
--- a/packages/core/src/utils/array.ts
+++ b/packages/core/src/utils/array.ts
@@ -54,6 +54,19 @@ export function findById<T extends { id: string }>(array: T[], id: string) {
return array.find((item) => item.id === id);
}
+export function findOrAdd<T>(
+ array: T[],
+ predicate: (item: T) => boolean,
+ item: T
+) {
+ const index = array.findIndex(predicate);
+ if (index === -1) {
+ array.push(item);
+ return item;
+ }
+ return array[index];
+}
+
export function hasItem<T>(array: T[], item: T) {
if (!array) return false;
return array.indexOf(item) > -1;
diff --git a/packages/core/src/utils/html-parser.ts b/packages/core/src/utils/html-parser.ts
index 09d955bb6..6ef352fae 100644
--- a/packages/core/src/utils/html-parser.ts
+++ b/packages/core/src/utils/html-parser.ts
@@ -131,11 +131,17 @@ export class HTMLParser {
}
}
-export function extractText(html: string) {
+export function extractText(html: string, retainTags?: string[]) {
let text = "";
const parser = new Parser(
{
- ontext: (data) => (text += data)
+ ontext: (data) => (text += data),
+ onopentag(name) {
+ if (retainTags?.includes(name)) text += `<${name}>`;
+ },
+ onclosetag(name) {
+ if (retainTags?.includes(name)) text += `</${name}>`;
+ }
},
{
lowerCaseTags: false
@@ -144,3 +150,80 @@ export function extractText(html: string) {
parser.end(html);
return text;
}
+
+const INLINE_TAGS = [
+ "a",
+ "abbr",
+ "acronym",
+ "b",
+ "bdo",
+ "big",
+ "br",
+ "button",
+ "cite",
+ "code",
+ "dfn",
+ "em",
+ "i",
+ "img",
+ "input",
+ "kbd",
+ "label",
+ "map",
+ "object",
+ "output",
+ "q",
+ "samp",
+ "script",
+ "select",
+ "small",
+ "span",
+ "strong",
+ "sub",
+ "sup",
+ "textarea",
+ "time",
+ "tt",
+ "var"
+];
+
+export function extractMatchingBlocks(html: string, matchTagName: string) {
+ const matches: string[] = [];
+ let text = "";
+ let openedTag: string | undefined = undefined;
+ let hasMatches = false;
+
+ const parser = new Parser(
+ {
+ ontext: (data) => (text += data),
+ onopentag(name) {
+ if (!INLINE_TAGS.includes(name) && name !== matchTagName) {
+ openedTag = name;
+ text = "";
+ hasMatches = false;
+ }
+ if (name === matchTagName) {
+ hasMatches = true;
+ text += `<${name}>`;
+ }
+ },
+ onclosetag(name) {
+ if (name === "br") text += "\n";
+ if (name === openedTag) {
+ if (hasMatches) matches.push(text);
+ text = "";
+ hasMatches = false;
+ openedTag = undefined;
+ }
+ if (name === matchTagName) text += `</${name}>`;
+ }
+ },
+ {
+ lowerCaseTags: false
+ }
+ );
+ parser.end(html);
+ if (hasMatches && text) matches.push(text);
+
+ return matches;
+}