Build your personal knowledge base with Trilium Notes
TypeScript electronelectron-appknowledge-baseknowledge-graphknowledge-managementknowledge-management-graphlocal-firstnote-managmentnote-takernote-takingnotebooknotesnotes-apppersonal-knowledge-basepersonal-wikiscriptableself-hostedself-hostingwiki
chore(share): build the export search index with node-html-parser
Elian Doran6e13699494e8
diff --git a/packages/trilium-core/package.json b/packages/trilium-core/package.jsonindex 562cd17e0e..69e12cad8d 100644--- a/packages/trilium-core/package.json+++ b/packages/trilium-core/package.json@@ -24,7 +24,6 @@"chardet": "2.2.0","color": "5.0.3","ejs": "7.0.2",- "html-to-text": "10.0.1","i18next": "26.5.10","js-yaml": "5.5.0","lz-string": "1.5.0",@@ -42,7 +41,6 @@},"devDependencies": {"@types/ejs": "3.1.5",- "@types/html-to-text": "9.0.4","@types/mime-types": "3.0.1","@types/sanitize-html": "2.16.2","@types/xml2js": "0.4.14"diff --git a/packages/trilium-core/src/services/export/zip/share_theme.spec.ts b/packages/trilium-core/src/services/export/zip/share_theme.spec.tsindex 46353f48e0..213f53eca9 100644--- a/packages/trilium-core/src/services/export/zip/share_theme.spec.ts+++ b/packages/trilium-core/src/services/export/zip/share_theme.spec.ts@@ -3,7 +3,6 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";// --- Mocks (hoisted above the module-under-test import) ---vi.mock("ejs", () => ({ default: { render: vi.fn(() => "<html>404</html>") } }));-vi.mock("html-to-text", () => ({ convert: vi.fn((s: string) => `TEXT:${s}`) }));const mockContentRenderer = {readShareTemplate: vi.fn(() => "TEMPLATE"),@@ -39,6 +38,7 @@ vi.mock("./share_data.js", async (importOriginal) => ({const {default: ShareThemeExportProvider,hasMermaidDiagrams,+ htmlToSearchText,getAppViewGroups,getShareThemeExportFiles,getShareThemeTranslationFiles@@ -174,7 +174,7 @@ describe("ShareThemeExportProvider", () => {const entry = (p as any).searchIndex.get("noteABC");expect(entry.title).toBe("Title");- expect(entry.content).toContain("TEXT:");+ expect(entry.content).toBe("html");// Path built from titles, excluding root and falsy titles.expect(entry.path).toBe("Parent");});@@ -419,6 +419,19 @@ describe("hasMermaidDiagrams", () => {});});+describe("htmlToSearchText", () => {+ it("joins the text of the blocks and drops scripts, styles and markup", () => {+ const html = `<h2>Hello & <b>World</b></h2><p>A <a href="https://x.y">link</a> here.</p>`+ + `<ul><li>one</li><li>two</li></ul><script>alert(1)</script><style>p {}</style>`+ + `<p>zerowidth and\tother\n\nspaces</p><pre>code\n block</pre>`;++ expect(htmlToSearchText(html))+ .toBe("Hello & World A link here. one two zero width and other spaces code block");+ expect(htmlToSearchText("a < b && c")).toBe("a < b && c");+ expect(htmlToSearchText("")).toBe("");+ });+});+describe("getAppViewGroups", () => {it("names the groups of the views, note types and drawn notes the subtree hosts", () => {const site = viewTree({ type: "text", children: [diff --git a/packages/trilium-core/src/services/export/zip/share_theme.ts b/packages/trilium-core/src/services/export/zip/share_theme.tsindex 7c5962d789..d1cd111d84 100644--- a/packages/trilium-core/src/services/export/zip/share_theme.ts+++ b/packages/trilium-core/src/services/export/zip/share_theme.ts@@ -2,8 +2,8 @@ import {getContentRendererGroup, getNoteContentType, getShareThemeGroupFiles, type ShareThemeManifest} from "@triliumnext/commons";import ejs from "ejs";-import { convert as convertToText } from "html-to-text";import { t } from "i18next";+import { parse } from "node-html-parser";import becca from "../../../becca/becca.js";import type BBranch from "../../../becca/entities/bbranch.js";@@ -74,9 +74,7 @@ export default class ShareThemeExportProvider extends ZipExportProvider {if (note) {// Prepare search index.- searchContent = typeof content === "string" ? convertToText(content, {- whitespaceCharacters: "\t\r\n\f "- }) : "";+ searchContent = typeof content === "string" ? htmlToSearchText(content) : "";// TODO: This will probably never match, but should it be exclude from running on code/jsFrontend notes?const getLink = (noteId: string) => (noteId === this.rootMeta?.noteId@@ -327,3 +325,15 @@ export function hasMermaidDiagrams(note: BNote) {}const MARKDOWN_MERMAID_FENCE = /^ {0,3}(`{3,}|~{3,})\s*mermaid\b/m;++/**+ * The text of `html` for the static search index: block boundaries become spaces, every run of+ * whitespace (zero-width spaces included) collapses to one, and scripts and styles are dropped.+ */+export function htmlToSearchText(html: string) {+ const root = parse(html);+ for (const element of root.querySelectorAll("script, style")) {+ element.remove();+ }+ return root.structuredText.replace(/[\s]+/g, " ").trim();+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 7222c6c6af..443ceff1c0 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1486,9 +1486,6 @@ importers:ejs:specifier: 7.0.2version: 7.0.2- html-to-text:- specifier: 10.0.1- version: 10.0.1i18next:specifier: 26.5.10version: 26.5.10([email protected])@@ -1535,9 +1532,6 @@ importers:'@types/ejs':specifier: 3.1.5version: 3.1.5- '@types/html-to-text':- specifier: 9.0.4- version: 9.0.4'@types/mime-types':specifier: 3.0.1version: 3.0.1@@ -4615,11 +4609,6 @@ packages:rollup:optional: true- '@selderee/[email protected]':- resolution: {integrity: sha512-oELmoyA6ML9jDRMV3kgcMQFKxUfBU0yFVn6yTctVaLT5ygXnxH52I3TZEgV9EhXJC68/uFvE5Daj1/25c0Xa/A==}- peerDependencies:- selderee: ~0.12.0-'@shikijs/[email protected]':resolution: {integrity: sha512-1nWINwKXxKKLqPibT5f4pAFLej9oZzQTsby8942OTlsJzOBZ0MWKiwzMsd+jhzu8YPCHAswGnnN1YtQfirL35g==}@@ -4866,9 +4855,6 @@ packages:'@types/[email protected]':resolution: {integrity: sha512-rp/ezSWaD1m44dPKICGhiskI13nVr7qTloFwDa/IYkhhf5nzwP+zIQcIJh3WIFSBOy/H1PzB40jPjMDksN4F+g==}- '@types/[email protected]':- resolution: {integrity: sha512-pUY3cKH/Nm2yYrEmDlPR1mR7yszjGx4DrwPjQ702C4/D5CwHuZTgZdIdwPkRbcuhs7BAh2L5rg3CL5cbRiGTCQ==}-'@types/[email protected]':resolution: {integrity: sha512-Wb1ymSAftCLxhc3D6vS0Ike/0xg7W6c+DQxAkerU6pD7C8CMzTYwvrwnlcrTfsVO/nMelB9KOKIT7+N5lOeQUg==}@@ -6836,10 +6822,6 @@ packages:resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==}- resolution: {integrity: sha512-uqbvqLUMrc6p0MO+WBRtTxY55hmyh94WRwI5a++PZe54X+bfVh59FSN7uWCBCW1CCVjzjnrwzfI8zidE2obMMw==}- engines: {node: '>=16.9.0'}-resolution: {integrity: sha512-3sUqbMEc77XqpdNO7FRyRog+eW3ph+GYCbj+rK+uYyRMuwsVy0rMiVtPn+QJlKFvWP/1PYpapqYn0Me2knFn+A==}engines: {node: '>=0.10.0'}@@ -7961,10 +7943,6 @@ packages:resolution: {integrity: sha512-n6l5uca7/y5joxZ3LUePhzmBFUJ+U2YWzhMa8XUTecSeSlQiZdF5XAd/Q3/WUl0VsXgUwWi8I7CNIwdI5WN1SQ==}engines: {node: '>=20.10'}- resolution: {integrity: sha512-GiVhRI1BatGARSCmlXWNCjDT0cWrwBWoeduLoV0WSKAgaV/wa+hUWy5LiQLUs4UwiUrE52ZCMfBGiKD87TDPrg==}- engines: {node: '>=20.19.0'}-resolution: {integrity: sha512-bEqo66MRXsUGxWHV5IP0PUiAWwoEjba4VCzg0LjFJBpchPaTfyfCKTG6bc5F8ucKec3q5y6qOdGyYTSBEvhCrg==}@@ -8654,9 +8632,6 @@ packages:resolution: {integrity: sha512-b94GiNHQNy6JNTrt5w6zNyffMrNkXZb3KTkCZJb2V1xaEGCk093vkZ2jk3tpaeP33/OiXC+WvK9AxUebnf5nbw==}engines: {node: '>= 0.6.3'}- resolution: {integrity: sha512-qMrZeyEekgdRQ9o6a4NAB2EQZrv827GJdn1vnapwSJ90hWRB4TzUSunvacPkxQ2TnNqHNI1/zSt0hlo0crG8Jw==}-resolution: {integrity: sha512-+bT2uH4E5LGE7h/n3evcS/sQlJXCpIp6ym8OWJ5eV6+67Dsql/LaaT7qJBAt2rzfoa/5QBGBhxDix1dMt2kQKQ==}engines: {node: '>= 0.8.0'}@@ -9679,9 +9654,6 @@ packages:resolution: {integrity: sha512-z1e/HMG90obSGeidlli3hj7cbocou0/wa5HacvI3ASx34PecNjNQeaHNo5WIZpWofN9kgkqV1q5YvXe3F0FoPw==}- resolution: {integrity: sha512-uNBJZzmb60l6p6VWLTmevizNAGnE0xoSf1n0B4q3ntegDNzcS68NRCcBDZTcyXHxt2XhBChsCuqj4M+nChvE/A==}-resolution: {integrity: sha512-CiyeOxFT/JZyN5m0z9PfXw4SCBJ6Sygz1Dpl0wqjlhDEGGBP1GnsUVEL0p63hoG1fcj3fHynXi9NYO4nWOL+qQ==}engines: {node: '>= 0.8'}@@ -9745,9 +9717,6 @@ packages:resolution: {integrity: sha512-nh39Mo1eGWmZS7y+mK/dQIqg7S1lp38DpRxkyoHf0ZcUs/HDc+yyTjuOtTvSMZHmfSLuSQaX945u05Y2Q6UWZg==}engines: {node: '>=14', npm: '>=7'}- resolution: {integrity: sha512-80B2AsU+I4Qdb0ZAPSfe9UwvGzwkM37IKIFEvdS3D/3Ndgv2bsuJ0bfG1+iEYO+l7Gfd4EUJmuRyq7efLgRMzQ==}-resolution: {integrity: sha512-F3asv42UuXchdzt+xXqfW1OGlVBe+mxa2mqI0pg5yAHZPvFmY3Y6drSf/GQ1A86WgWEN9Kzh/WrgKa6iGcHXLg==}@@ -10444,9 +10413,6 @@ packages:resolution: {integrity: sha512-AckIIV90rPDcBcglUwXPF3kg0P0qmPsPXAj6BBEENQE1p5yA1xfmDJzfi1Tappj37Pv2mVbKpL3Z1T+Nn7k1Qw==}- resolution: {integrity: sha512-b1YMh3+DHZp59DLna3qVwQ5iOla/nrI6mLBNW02XxU77M3046Df6VLkoaJyFz20VsGIG5kkp+FK0kg4K4HnUFw==}-resolution: {integrity: sha512-cBznnQ9KjJqU67B52RMC65CMarK2600WFnbkcaiwWq3xy/5haFJlshgnpjovMVJ+Hff49d8GEn0b87C5pDQ10g==}hasBin: true@@ -15582,12 +15548,6 @@ snapshots:estree-walker: 2.0.2picomatch: 4.0.7- '@selderee/[email protected]([email protected])':- dependencies:- domelementtype: 2.3.0- domhandler: 5.0.3- selderee: 0.12.0-'@shikijs/[email protected]':dependencies:'@shikijs/types': 3.23.0@@ -15876,8 +15836,6 @@ snapshots:dependencies:'@types/unist': 3.0.3- '@types/[email protected]': {}-'@types/[email protected]': {}'@types/[email protected]': {}@@ -18450,8 +18408,6 @@ snapshots:- [email protected]: {}-@@ -19951,14 +19907,6 @@ snapshots:- dependencies:- '@selderee/plugin-htmlparser2': 0.12.0([email protected])- deepmerge-ts: 8.0.2- dom-serializer: 2.0.0- htmlparser2: 10.1.0- selderee: 0.12.0-[email protected](patch_hash=8acc288f8e9cde3c8a0c6e89f0b3796aeeb8251b88db9d1ba313e2352a591d68):@@ -20673,8 +20621,6 @@ snapshots:dependencies:readable-stream: 2.3.8- [email protected]: {}-dependencies:prelude-ls: 1.2.1@@ -21863,11 +21809,6 @@ snapshots:dependencies:entities: 8.1.0- dependencies:- leac: 0.7.0- peberminta: 0.10.0-@@ -21914,8 +21855,6 @@ snapshots:- [email protected]: {}-@@ -22728,10 +22667,6 @@ snapshots:- dependencies:- parseley: 0.13.1-optional: true