A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: add support for highlighting search results
Abdullah Atta66041a45f6e6
diff --git a/packages/core/src/api/lookup.ts b/packages/core/src/api/lookup.tsindex d2c9d56eb..533677c26 100644--- a/packages/core/src/api/lookup.ts+++ b/packages/core/src/api/lookup.ts@@ -20,7 +20,9 @@ along with this program. If not, see <http://www.gnu.org/licenses/>.import { match } from "fuzzyjs";import Database from "./index.js";import {+ HighlightedResult,Item,+ Match,Note,Notebook,Reminder,@@ -36,7 +38,8 @@ import { rebuildSearchIndex } from "../database/fts.js";import { transformQuery } from "../utils/query-transformer.js";import { getSortSelectors, groupArray } from "../utils/grouping.js";import { fuzzy } from "../utils/fuzzy.js";-import { extractText } from "../utils/html-parser.js";+import { extractMatchingBlocks, extractText } from "../utils/html-parser.js";+import { findOrAdd } from "../utils/array.js";type SearchResults<T> = {sorted: (sortOptions?: SortOptions) => Promise<VirtualizedGrouping<T>>;@@ -53,100 +56,170 @@ type FuzzySearchField<T> = {export default class Lookup {constructor(private readonly db: Database) {}- notes(query: string, notes?: FilteredSelector<Note>): SearchResults<Note> {- return this.toSearchResults(async (sortOptions) => {- const db = this.db.sql() as unknown as Kysely<RawDatabaseSchema>;- const excludedIds = this.db.trash.cache.notes;-- const { query: transformedQuery, tokens } = transformQuery(query);-- const resultsA: string[] =- transformedQuery.length === 0- ? []- : await db- .selectFrom((eb) =>- eb- .selectFrom("notes_fts")- .$if(!!notes, (eb) =>- eb.where("id", "in", notes!.filter.select("id"))- )- .$if(excludedIds.length > 0, (eb) =>- eb.where("id", "not in", excludedIds)- )- .where("title", "match", transformedQuery)- .select(["id", sql<number>`rank * 10`.as("rank")])- .unionAll((eb) =>- eb- .selectFrom("content_fts")- .$if(!!notes, (eb) =>- eb.where("noteId", "in", notes!.filter.select("id"))- )- .$if(excludedIds.length > 0, (eb) =>- eb.where("noteId", "not in", excludedIds)- )- .where("data", "match", transformedQuery)- .select(["noteId as id", "rank"])- .$castTo<{- id: string;- rank: number;- }>()- )- .as("results")- )- .select(["results.id"])- .groupBy("results.id")- .orderBy(- sql`SUM(results.rank)`,- sortOptions?.sortDirection || "desc"- )- .execute()- .catch((e) => {- logger.error(e, `Error while searching`, { query });- return [];- })- .then((r) => r.map((r) => r.id));-- const smallTokens = Array.from(- new Set(- tokens.filter((token) => token.length < 3 && token !== "OR")- ).values()- );- if (smallTokens.length === 0) return resultsA;+ async notes(+ query: string,+ notes?: FilteredSelector<Note>+ ): Promise<VirtualizedGrouping<HighlightedResult>> {+ const db = this.db.sql() as unknown as Kysely<RawDatabaseSchema>;+ const excludedIds = this.db.trash.cache.notes;- const results = [];+ const { query: transformedQuery, tokens } = transformQuery(query);- const titles = await db- .selectFrom("notes")- .$if(!!transformedQuery && resultsA.length > 0, (eb) =>- eb.where("id", "in", resultsA)+ let mergedResults: HighlightedResult[] = [];+ if (transformedQuery.length > 0) {+ const results = await db+ .selectFrom((eb) =>+ eb+ .selectFrom("notes_fts")+ .$if(!!notes, (eb) =>+ eb.where("id", "in", notes!.filter.select("id"))+ )+ .$if(excludedIds.length > 0, (eb) =>+ eb.where("id", "not in", excludedIds)+ )+ .where("title", "match", transformedQuery)+ .select([+ "id",+ sql<string>`'title'`.as("type"),+ sql<string>`highlight(notes_fts, 1, '<nnmark>', '</nnmark>')`.as(+ "match"+ ),+ sql<number>`rank * 10`.as("rank")+ ])+ .unionAll((eb) =>+ eb+ .selectFrom("content_fts")+ .$if(!!notes, (eb) =>+ eb.where("noteId", "in", notes!.filter.select("id"))+ )+ .$if(excludedIds.length > 0, (eb) =>+ eb.where("noteId", "not in", excludedIds)+ )+ .where("data", "match", transformedQuery)+ .select([+ "noteId as id",+ sql<string>`'content'`.as("type"),+ sql<string>`highlight(content_fts, 2, '<nnmark>', '</nnmark>')`.as(+ "match"+ ),+ "rank"+ ])+ .$castTo<{+ id: string;+ type: string;+ rank: number;+ match: string;+ }>()+ )+ .as("results"))- .select(["id", "title"])- .execute();+ .select(["results.id", "results.match", "results.type", "results.rank"])+ .execute()+ .catch((e) => {+ logger.error(e, `Error while searching`, { query });+ return [];+ });- const htmls = await db- .selectFrom("content")- .$if(!!transformedQuery && resultsA.length > 0, (eb) =>- eb.where("noteId", "in", resultsA)- )- .select(["data", "noteId as id"])- .$castTo<{ data: string; id: string }>()- .execute();-- for (let i = 0; i < titles.length; i++) {- const title = titles[i];- const html = htmls.find((h) => h.id === title.id);- const text = html ? extractText(html.data) : "";-- if (- smallTokens.every((token) => !!title.title?.includes(token)) ||- smallTokens.every((token) => !!text?.includes(token))- ) {- results.push(title.id);+ for (const result of results) {+ const old = findOrAdd(mergedResults, (r) => r.id === result.id, {+ type: "searchResult",+ id: result.id,+ content: [],+ title: []+ });++ if (result.type === "content")+ old.content = extractMatchingBlocks(result.match, "nnmark").flatMap(+ (block) => {+ return splitHighlightedMatch(block);+ }+ );+ if (result.type === "title")+ old.title = splitHighlightedMatch(result.match).flatMap((m) => m);+ }++ const resultsWithMissingTitle = mergedResults+ .filter((r) => !r.title.length)+ .map((r) => r.id);++ if (resultsWithMissingTitle.length > 0) {+ const titles = await db+ .selectFrom("notes")+ .where("id", "in", resultsWithMissingTitle)+ .select(["id", "title"])+ .execute();+ for (const title of titles) {+ const result = mergedResults.find((r) => r.id === title.id);+ if (!result || !title.title) continue;+ result.title = stringToMatch(title.title);}}- return results;- }, notes || this.db.notes.all);+ mergedResults = mergedResults.filter((r) => !!r.title);+ }++ const smallTokens = Array.from(+ new Set(+ tokens.filter((token) => token.length < 3 && token !== "OR")+ ).values()+ );++ if (smallTokens.length === 0)+ return arrayToVirtualizedGrouping(+ mergedResults,+ this.db.options.batchSize+ );++ const ids = mergedResults.map((r) => r.id);+ const titles = await db+ .selectFrom("notes")+ .$if(!!transformedQuery && ids.length > 0, (eb) =>+ eb.where("id", "in", ids)+ )+ .select(["id", "title"])+ .execute();++ const htmls = await db+ .selectFrom("content")+ .where("content.locked", "!=", true)+ .$if(!!transformedQuery && ids.length > 0, (eb) =>+ eb.where("noteId", "in", ids)+ )+ .select(["data", "noteId as id"])+ .$castTo<{ data: string; id: string }>()+ .execute();++ for (let i = 0; i < titles.length; i++) {+ const title = titles[i];+ const result = findOrAdd(mergedResults, (r) => r.id === title.id, {+ id: title.id,+ title: stringToMatch(title.title || ""),+ type: "searchResult",+ content: []+ });++ const html = htmls.find((h) => h.id === title.id);+ const text = html ? extractText(html.data) : "";++ if (+ title.title &&+ smallTokens.every((token) => !!title.title?.includes(token))+ ) {+ result.title.push(+ ...splitHighlightedMatch(+ highlightQueries(title.title, smallTokens)+ ).flatMap((m) => m)+ );+ }++ if (text && smallTokens.every((token) => !!text?.includes(token))) {+ result.content.push(+ ...splitHighlightedMatch(highlightQueries(text, smallTokens))+ );+ }+ }++ return arrayToVirtualizedGrouping(mergedResults, this.db.options.batchSize);}notebooks(query: string) {@@ -332,3 +405,219 @@ export default class Lookup {await rebuildSearchIndex(db);}}++function highlightQueries(text: string, queries: string[]): string {+ if (!text || !queries.length) return text;++ // Collect all ranges+ const ranges = [];+ const lowerText = text.toLowerCase();++ for (let i = 0; i < queries.length; i++) {+ const query = queries[i];+ const lowerQuery = query.toLowerCase();+ const queryLen = query.length;+ let pos = 0;++ while ((pos = lowerText.indexOf(lowerQuery, pos)) !== -1) {+ ranges.push({+ start: pos,+ end: pos + queryLen,+ len: queryLen+ });+ pos += 1;+ }+ }++ if (!ranges.length) return text;++ // Sort by start position, then by length (longer first)+ ranges.sort((a, b) => a.start - b.start || b.len - a.len);++ // Filter overlaps and merge adjacent ranges+ const merged = [ranges[0]];+ for (let i = 1; i < ranges.length; i++) {+ const current = ranges[i];+ const previous = merged[merged.length - 1];++ if (current.start > previous.end) {+ // No overlap or adjacency - add as new range+ merged.push(current);+ } else if (current.start === previous.end) {+ // Adjacent ranges - merge them+ previous.end = current.end;+ previous.len = previous.end - previous.start;+ }+ // Overlapping ranges are skipped+ }++ // Build result using array of parts+ const parts = [];+ let lastEnd = 0;++ for (const { start, end } of merged) {+ if (start > lastEnd) {+ parts.push(text.slice(lastEnd, start));+ }+ parts.push("<nnmark>", text.slice(start, end), "</nnmark>");+ lastEnd = end;+ }++ if (lastEnd < text.length) {+ parts.push(text.slice(lastEnd));+ }++ return parts.join("");+}++function arrayToVirtualizedGrouping<T extends { id: string }>(+ array: T[],+ batchSize: number+): VirtualizedGrouping<T> {+ return new VirtualizedGrouping<T>(+ array.length,+ batchSize,+ () => Promise.resolve(array.map((c) => c.id)),+ async (start, end) => {+ const items = array.slice(start, end);+ return {+ ids: items.map((i) => i.id),+ items+ };+ },+ (items) => groupArray(items, () => `${items.length} results`)+ );+}++function splitHighlightedMatch(text: string): Match[][] {+ const parts = text.split(/<nnmark>(.*?)<\/nnmark>/g);+ const allMatches: Match[][] = [];+ let matches: Match[] = [];+ let totalLength = 0;++ for (let i = 0; i < parts.length - 1; i += 2) {+ const prefix = parts[i];+ const match = parts[i + 1];+ // const suffix = parts[i + 2] || "";+ const matchLength = prefix.length + match.length; // + suffix.length;++ if (totalLength > 120 && matches.length > 0) {+ matches[matches.length - 1].suffix += "...";+ allMatches.push(matches);+ matches = [];+ totalLength = 0;+ }++ matches.push({+ match,+ prefix,+ suffix: ""+ });++ totalLength += matchLength;+ }++ if (matches.length > 0) {+ matches[matches.length - 1].suffix += parts[parts.length - 1];+ allMatches.push(matches);+ }++ for (const matches of allMatches) {+ const totalLength = matches.reduce(+ (length, curr) =>+ length + curr.match.length + curr.prefix.length + curr.suffix.length,+ 0+ );+ if (totalLength > 200) {+ const start = matches[0];+ const end = matches[matches.length - 1];++ const centered = centerMatch(+ start.prefix,+ end.suffix,+ totalLength - (start.prefix.length + end.suffix.length),+ {+ maxLength: 200+ }+ );++ start.prefix = centered.prefix || " ";+ end.suffix = centered.suffix || " ";+ }+ }++ return allMatches;+}++interface CenterOptions {+ maxLength?: number; // Maximum total length of output+ minContext?: number; // Minimum context on each side+ ellipsis?: string; // String to use for truncation+ preferLeft?: boolean; // Prefer more context on left when odd length+}++function centerMatch(+ prefix: string,+ suffix: string,+ matchLength: number,+ options: CenterOptions = {}+): { prefix?: string; suffix?: string } {+ const {+ maxLength = 120,+ minContext = 20,+ ellipsis = "...",+ preferLeft = true+ } = options;++ // Handle edge cases+ if (!match) return {};+ if (matchLength >= maxLength) return {};++ // Calculate available space for context+ const availableSpace = maxLength - matchLength;++ // Calculate initial context lengths+ let leftLength = Math.floor(availableSpace / 2);+ let rightLength = availableSpace - leftLength;++ // Adjust if we prefer left context+ if (preferLeft && availableSpace % 2 !== 0) {+ leftLength++;+ rightLength--;+ }++ // Ensure minimum context if possible+ if (leftLength < minContext && prefix.length > leftLength) {+ const diff = Math.min(rightLength - minContext, minContext - leftLength);+ if (diff > 0) {+ leftLength += diff;+ rightLength -= diff;+ }+ } else if (rightLength < minContext && suffix.length > rightLength) {+ const diff = Math.min(leftLength - minContext, minContext - rightLength);+ if (diff > 0) {+ rightLength += diff;+ leftLength -= diff;+ }+ }++ // Build result+ const left =+ prefix.length > leftLength ? ellipsis + prefix.slice(-leftLength) : prefix;+ const right =+ suffix.length > rightLength+ ? suffix.slice(0, rightLength) + ellipsis+ : suffix;++ return { prefix: left, suffix: right };+}++function stringToMatch(str: string): Match[] {+ return [+ {+ prefix: str,+ match: "",+ suffix: ""+ }+ ];+}diff --git a/packages/core/src/types.ts b/packages/core/src/types.tsindex e9f6cbe3f..2d8a5df2c 100644--- a/packages/core/src/types.ts+++ b/packages/core/src/types.ts@@ -488,6 +488,18 @@ export interface Vault extends BaseItem<"vault"> {key: Cipher<"base64">;}+export type Match = {+ prefix: string;+ match: string;+ suffix: string;+};+export interface HighlightedResult {+ type: "searchResult";+ id: string;+ title: Match[];+ content: Match[][];+}+export interface DeletedItem {id: string;deleted: true;diff --git a/packages/core/src/utils/array.ts b/packages/core/src/utils/array.tsindex d274322d0..8d92d52d0 100644--- a/packages/core/src/utils/array.ts+++ b/packages/core/src/utils/array.ts@@ -54,6 +54,19 @@ export function findById<T extends { id: string }>(array: T[], id: string) {return array.find((item) => item.id === id);}+export function findOrAdd<T>(+ array: T[],+ predicate: (item: T) => boolean,+ item: T+) {+ const index = array.findIndex(predicate);+ if (index === -1) {+ array.push(item);+ return item;+ }+ return array[index];+}+export function hasItem<T>(array: T[], item: T) {if (!array) return false;return array.indexOf(item) > -1;diff --git a/packages/core/src/utils/html-parser.ts b/packages/core/src/utils/html-parser.tsindex 09d955bb6..6ef352fae 100644--- a/packages/core/src/utils/html-parser.ts+++ b/packages/core/src/utils/html-parser.ts@@ -131,11 +131,17 @@ export class HTMLParser {}}-export function extractText(html: string) {+export function extractText(html: string, retainTags?: string[]) {let text = "";const parser = new Parser({- ontext: (data) => (text += data)+ ontext: (data) => (text += data),+ onopentag(name) {+ if (retainTags?.includes(name)) text += `<${name}>`;+ },+ onclosetag(name) {+ if (retainTags?.includes(name)) text += `</${name}>`;+ }},{lowerCaseTags: false@@ -144,3 +150,80 @@ export function extractText(html: string) {parser.end(html);return text;}++const INLINE_TAGS = [+ "a",+ "abbr",+ "acronym",+ "b",+ "bdo",+ "big",+ "br",+ "button",+ "cite",+ "code",+ "dfn",+ "em",+ "i",+ "img",+ "input",+ "kbd",+ "label",+ "map",+ "object",+ "output",+ "q",+ "samp",+ "script",+ "select",+ "small",+ "span",+ "strong",+ "sub",+ "sup",+ "textarea",+ "time",+ "tt",+ "var"+];++export function extractMatchingBlocks(html: string, matchTagName: string) {+ const matches: string[] = [];+ let text = "";+ let openedTag: string | undefined = undefined;+ let hasMatches = false;++ const parser = new Parser(+ {+ ontext: (data) => (text += data),+ onopentag(name) {+ if (!INLINE_TAGS.includes(name) && name !== matchTagName) {+ openedTag = name;+ text = "";+ hasMatches = false;+ }+ if (name === matchTagName) {+ hasMatches = true;+ text += `<${name}>`;+ }+ },+ onclosetag(name) {+ if (name === "br") text += "\n";+ if (name === openedTag) {+ if (hasMatches) matches.push(text);+ text = "";+ hasMatches = false;+ openedTag = undefined;+ }+ if (name === matchTagName) text += `</${name}>`;+ }+ },+ {+ lowerCaseTags: false+ }+ );+ parser.end(html);+ if (hasMatches && text) matches.push(text);++ return matches;+}