A fully open source & end-to-end encrypted note taking alternative to Evernote.
TypeScript asp-net-coredotnet-coreelectronfosshacktoberfestnote-managmentnote-takingnote-taking-appnotebooknotesnotes-appopen-sourceopensourceprivacyproductivityreactreact-nativereactjsself-hosted
core: fix string and html helper edge cases (#10534)
Abdullah Atta061730802958
diff --git a/packages/core/__tests__/migrations.test.ts b/packages/core/__tests__/migrations.test.tsindex 4de149795..4ec8dee28 100644--- a/packages/core/__tests__/migrations.test.ts+++ b/packages/core/__tests__/migrations.test.ts@@ -89,6 +89,19 @@ test("[5.3] decode wrapped table html entities", () =>);}));+test("[5.3] decode only the wrapped table html entities", () =>+ databaseTest().then(async (db) => {+ const item = {+ type: "tiny",+ dateEdited: Date.now(),+ data: `<div class="table-container"><table></table></div><p><b></p><div class="table-container"><table></table></div>`+ };+ expect(await migrateItem(item, 5.3, 5.4, "tiny", db, "local")).toBe(true);+ expect(item.data).toBe(+ `<div class="table-container"><table></table></div><p><b></p><div class="table-container"><table></table></div>`+ );+ }));+describe.concurrent("[5.4] convert tiny to tiptap", () => {const cases = [{@@ -106,6 +119,11 @@ describe.concurrent("[5.4] convert tiny to tiptap", () => {from: `<p><img src="hello.jpg" /></p>`,to: `<img src="hello.jpg">`},+ {+ name: "keep text around images when moving them out of paragraphs",+ from: `<p>before<img src="hello.jpg" />after</p>`,+ to: `<p>before</p><img src="hello.jpg"><p>after</p>`+ },{name: "replace [data-mce-bogus] elements",from: `<p><br data-mce-bogus="" /></p><br data-mce-bogus="" />`,diff --git a/packages/core/src/content-types/__tests__/tiptap.test.js b/packages/core/src/content-types/__tests__/tiptap.test.jsindex 2b6923f18..13aa7ad99 100644--- a/packages/core/src/content-types/__tests__/tiptap.test.js+++ b/packages/core/src/content-types/__tests__/tiptap.test.js@@ -92,3 +92,21 @@ for (const html in HTMLS) {expect(tiptap.toTXT()).toMatchSnapshot(`html-to-txt-${html}.txt`);});}++test("search ignores empty tokens", () => {+ const tiptap = new Tiptap("<p>hello world</p>");+ expect(tiptap.search("zzz ")).toBe(false);+ expect(tiptap.search(" hello")).toBe(true);+});++test("list blocks include their nested lists' content", () => {+ // the editor only assigns block ids to top-level blocks so nested lists have none+ const tiptap = new Tiptap(+ `<ul data-block-id="ul1"><li><p>parent</p><ul><li><p>child</p></li></ul></li><li><p>sibling</p></li></ul>`+ );+ const blocks = tiptap.extract("blocks").blocks;+ expect(blocks).toHaveLength(1);+ expect(blocks[0].content).toContain("parent");+ expect(blocks[0].content).toContain("child");+ expect(blocks[0].content).toContain("sibling");+});diff --git a/packages/core/src/content-types/tiptap.ts b/packages/core/src/content-types/tiptap.tsindex bcfbd3b4e..cd35f074d 100644--- a/packages/core/src/content-types/tiptap.ts+++ b/packages/core/src/content-types/tiptap.ts@@ -102,7 +102,7 @@ export class Tiptap {// }search(query: string) {- const tokens = query.toLowerCase().split(splitter);+ const tokens = query.toLowerCase().split(splitter).filter(Boolean);const lowercase = this.toTXT().toLowerCase();return tokens.some((token) => lowercase.indexOf(token) > -1);}diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.tsindex dd9734f42..8820f129f 100644--- a/packages/core/src/migrations.ts+++ b/packages/core/src/migrations.ts@@ -657,7 +657,8 @@ function removeToxClassFromChecklist(html: string): string {: document.body.innerHTML;}-const regex = /<div class="table-container".*<\/table><\/div>/gm;+const regex =+ /<div class="table-container".*?<\/table><\/div>/gm;function decodeWrappedTableHtml(html: string) {return html.replace(regex, (match) => {const html = decodeHTML5(match);@@ -669,6 +670,12 @@ const NEWLINE_REPLACEMENT_REGEX = /\n|<br>|<br\/>/gm;const PREBLOCK_REGEX = /(<pre.*?>)(.*?)(<\/pre>)/gm;const SPAN_REGEX = /<span class=.*?>(.*?)<\/span>/gm;+function isEmpty(node: Node) {+ return Array.from(node.childNodes).every(+ (child) => child.nodeType === 3 && !child.textContent?.trim()+ );+}+export function tinyToTiptap(html: string) {if (typeof html !== "string") return html;@@ -703,7 +710,15 @@ export function tinyToTiptap(html: string) {const images = document.querySelectorAll("p > img");for (const image of images) {- image.parentElement?.replaceWith(image.cloneNode());+ const paragraph = image.parentElement;+ if (!paragraph) continue;++ // split the paragraph around the image so sibling content is kept+ const after = paragraph.cloneNode(false);+ while (image.nextSibling) after.appendChild(image.nextSibling);+ paragraph.after(image);+ if (!isEmpty(after)) image.after(after);+ if (isEmpty(paragraph)) paragraph.remove();}const bogus = document.querySelectorAll("[data-mce-bogus]");diff --git a/packages/core/src/utils/__tests__/content-block.test.ts b/packages/core/src/utils/__tests__/content-block.test.tsindex 8c332312d..e549c5809 100644--- a/packages/core/src/utils/__tests__/content-block.test.ts+++ b/packages/core/src/utils/__tests__/content-block.test.ts@@ -39,36 +39,36 @@ describe("ContentBlock Utils", () => {expect(links[0].text).toBe("link");});- it("should return an empty array if no internal links are present", () => {+ it("should handle link text containing |", () => {const block: ContentBlock = {type: "someType",id: "someId",- content: "This is a test with no links"+ content: "This is a test [[nn://note/123|a|b]]"};const links = extractInternalLinks(block);- expect(links).toHaveLength(0);+ expect(links).toHaveLength(1);+ expect(links[0].id).toBe("123");+ expect(links[0].text).toBe("a|b");});- it("should skip links with no noteId", () => {+ it("should return an empty array if no internal links are present", () => {const block: ContentBlock = {type: "someType",id: "someId",- content: "This is a test [[nn://note/|link]] with undefined URL"+ content: "This is a test with no links"};const links = extractInternalLinks(block);expect(links).toHaveLength(0);});- it("should extract internal links with '|' in id", () => {+ it("should skip links with no noteId", () => {const block: ContentBlock = {type: "someType",id: "someId",- content: "This is a test [[nn://note/myid|ofmyid|actualtext]]"+ content: "This is a test [[nn://note/|link]] with undefined URL"};const links = extractInternalLinks(block);- expect(links).toHaveLength(1);- expect(links[0].id).toBe("myid|ofmyid");- expect(links[0].text).toBe("actualtext");+ expect(links).toHaveLength(0);});});diff --git a/packages/core/src/utils/__tests__/fuzzy.test.ts b/packages/core/src/utils/__tests__/fuzzy.test.tsindex 5b576ffe3..d5c544540 100644--- a/packages/core/src/utils/__tests__/fuzzy.test.ts+++ b/packages/core/src/utils/__tests__/fuzzy.test.ts@@ -130,3 +130,10 @@ describe("lookup.fuzzy", () => {});});});++test("sanitized query is tried when the raw query has no matches", () => {+ const items = [{ id: "1", title: "hello world" }];+ expect(fuzzy("hello-world", items, (i) => i.id, { title: 1 })).toStrictEqual(+ items+ );+});diff --git a/packages/core/src/utils/__tests__/html-rewriter.test.ts b/packages/core/src/utils/__tests__/html-rewriter.test.tsnew file mode 100644index 000000000..d228f74d2--- /dev/null+++ b/packages/core/src/utils/__tests__/html-rewriter.test.ts@@ -0,0 +1,37 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { test, expect } from "vitest";+import { HTMLRewriter } from "../html-rewriter.js";++test("implied close of a non-void element doesn't write /> into text", () => {+ expect(new HTMLRewriter().transform("<p>a<p>b")).toBe("<p>a</p><p>b</p>");+});++test("void and self-closing elements are still self-closed", () => {+ expect(new HTMLRewriter().transform('<p>a<img src="x"><br/>b</p>')).toBe(+ '<p>a<img src="x"/><br/>b</p>'+ );+});++test("attribute values containing quotes are escaped", () => {+ expect(new HTMLRewriter().transform(`<p title='say "hi"'>a</p>`)).toBe(+ '<p title="say "hi"">a</p>'+ );+});diff --git a/packages/core/src/utils/__tests__/templates.test.ts b/packages/core/src/utils/__tests__/templates.test.tsnew file mode 100644index 000000000..5887691b1--- /dev/null+++ b/packages/core/src/utils/__tests__/templates.test.ts@@ -0,0 +1,34 @@+/*+This file is part of the Notesnook project (https://notesnook.com/)++Copyright (C) 2023 Streetwriters (Private) Limited++This program is free software: you can redistribute it and/or modify+it under the terms of the GNU General Public License as published by+the Free Software Foundation, either version 3 of the License, or+(at your option) any later version.++This program is distributed in the hope that it will be useful,+but WITHOUT ANY WARRANTY; without even the implied warranty of+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the+GNU General Public License for more details.++You should have received a copy of the GNU General Public License+along with this program. If not, see <http://www.gnu.org/licenses/>.+*/++import { test, expect } from "vitest";+import { templateWithFrontmatter } from "../templates/md.js";++test("frontmatter quotes tags and color", () => {+ const md = templateWithFrontmatter({+ title: "Note",+ content: "",+ dateCreated: 0,+ dateEdited: 0,+ color: "red: #f00",+ tags: ["a: b", "#c"]+ } as any);+ expect(md).toContain('color: "red: #f00"');+ expect(md).toContain('tags: "a: b, #c"');+});diff --git a/packages/core/src/utils/__tests__/title-format.test.js b/packages/core/src/utils/__tests__/title-format.test.jsindex d34df0c63..abfc1dd6f 100644--- a/packages/core/src/utils/__tests__/title-format.test.js+++ b/packages/core/src/utils/__tests__/title-format.test.js@@ -78,3 +78,16 @@ describe("day format", () => {).toBe("Note Wed");});});++test("$headline$ with replacement patterns is inserted literally", () => {+ expect(+ formatTitle(+ "Note $headline$",+ "[DD-MM-YYYY]",+ "[hh:mm]",+ "short",+ "costs $& more $$ $'",+ 0+ )+ ).toBe("Note costs $& more $$ $'");+});diff --git a/packages/core/src/utils/content-block.ts b/packages/core/src/utils/content-block.tsindex d77c97c65..fee3617a0 100644--- a/packages/core/src/utils/content-block.ts+++ b/packages/core/src/utils/content-block.ts@@ -27,8 +27,8 @@ export function extractInternalLinks(block: ContentBlock) {const links: InternalLinkWithOffset[] = [];for (const match of matches) {if (match.index === undefined) continue;- const url = match[1].slice(0, match[1].lastIndexOf("|"));- const text = match[1].slice(match[1].lastIndexOf("|") + 1);+ const url = match[1].slice(0, match[1].indexOf("|"));+ const text = match[1].slice(match[1].indexOf("|") + 1);const link = parseInternalLink(url);if (!link) continue;links.push({diff --git a/packages/core/src/utils/fuzzy.ts b/packages/core/src/utils/fuzzy.tsindex d51b25c24..044f7f3d2 100644--- a/packages/core/src/utils/fuzzy.ts+++ b/packages/core/src/utils/fuzzy.ts@@ -32,7 +32,6 @@ export function fuzzy<T>(} = {}): T[] {const results = fuzzyMatch(query, items, getIdentifier, fields, options);- if (results.size === 0) return [];// if the query contains spaces & other non-alphanumeric characters, then we// should search again with a sanitized query to catch more matches.diff --git a/packages/core/src/utils/html-rewriter.ts b/packages/core/src/utils/html-rewriter.tsindex cfea11976..0d88f366b 100644--- a/packages/core/src/utils/html-rewriter.ts+++ b/packages/core/src/utils/html-rewriter.ts@@ -86,7 +86,7 @@ export class HTMLRewriter {if (attr) {for (const key in attr) {if (!key) continue;- this.write(` ${key}="${attr[key]}"`);+ this.write(` ${key}="${attr[key].replace(/"/g, """)}"`);}}this.currentTag = name;@@ -104,9 +104,12 @@ export class HTMLRewriter {if (this.skippedTags.pop()) return;- if (!isImplied) this.closeTag();+ // an implied close right after the open tag is a void or+ // self-closing element; otherwise the element had content.+ const selfClose = isImplied && this.currentTag !== null;+ if (!selfClose) this.closeTag();- this.write(isImplied ? "/>" : `</${name}>`);+ this.write(selfClose ? "/>" : `</${name}>`);if (this.currentTag) {this.currentTag = null;diff --git a/packages/core/src/utils/templates/html/index.ts b/packages/core/src/utils/templates/html/index.tsindex 172d4252c..fa9922c63 100644--- a/packages/core/src/utils/templates/html/index.ts+++ b/packages/core/src/utils/templates/html/index.ts@@ -97,6 +97,9 @@ async function preprocessHTML(templateData: TemplateData) {const codeblocks = doc.querySelectorAll("pre > code");if (codeblocks.length) {+ // stop Prism from auto-highlighting the host document on import+ const globalScope = globalThis as { Prism?: { manual?: boolean } };+ if (!globalScope.Prism) globalScope.Prism = { manual: true };const { default: prismjs } = await import("prismjs");// const { loadLanguage } = await import("./languages/index.js");(prismjs as any).register = (syntax: (syntax: any) => void) => {diff --git a/packages/core/src/utils/templates/md.ts b/packages/core/src/utils/templates/md.tsindex 5b096a6c5..feb54bf5e 100644--- a/packages/core/src/utils/templates/md.ts+++ b/packages/core/src/utils/templates/md.ts@@ -41,7 +41,7 @@ function buildFrontmatter(data: TemplateData) {if (data.pinned) lines.push(`pinned: ${data.pinned}`);if (data.favorite) lines.push(`favorite: ${data.favorite}`);if (data.archived) lines.push(`archived: ${data.archived}`);- if (data.color) lines.push(`color: ${data.color}`);- if (data.tags) lines.push(`tags: ${data.tags.join(", ")}`);+ if (data.color) lines.push(`color: ${JSON.stringify(data.color)}`);+ if (data.tags) lines.push(`tags: ${JSON.stringify(data.tags.join(", "))}`);return lines.join("\n");}diff --git a/packages/core/src/utils/title-format.ts b/packages/core/src/utils/title-format.tsindex ce3e41a69..a3b277b0c 100644--- a/packages/core/src/utils/title-format.ts+++ b/packages/core/src/utils/title-format.ts@@ -64,7 +64,7 @@ export function formatTitle(.replace(DATE_REGEX, date).replace(TIME_REGEX, time).replace(DAY_REGEX, day)- .replace(HEADLINE_REGEX, headline || "")+ .replace(HEADLINE_REGEX, () => headline || "").replace(TIMESTAMP_REGEX, timestamp).replace(TIMESTAMP_Z_REGEX, timestampWithTimeZone).replace(COUNT_REGEX, `${totalNotes + 1}`);