Skip to content

A fully open source & end-to-end encrypted note taking alternative to Evernote.

core: fix string and html helper edge cases (#10534)

Abdullah Atta061730802958
diff --git a/packages/core/__tests__/migrations.test.ts b/packages/core/__tests__/migrations.test.ts
index 4de149795..4ec8dee28 100644
--- a/packages/core/__tests__/migrations.test.ts
+++ b/packages/core/__tests__/migrations.test.ts
@@ -89,6 +89,19 @@ test("[5.3] decode wrapped table html entities", () =>
);
}));
+test("[5.3] decode only the wrapped table html entities", () =>
+ databaseTest().then(async (db) => {
+ const item = {
+ type: "tiny",
+ dateEdited: Date.now(),
+ data: `&lt;div class="table-container"&gt;&lt;table&gt;&lt;/table&gt;&lt;/div&gt;<p>&lt;b&gt;</p>&lt;div class="table-container"&gt;&lt;table&gt;&lt;/table&gt;&lt;/div&gt;`
+ };
+ expect(await migrateItem(item, 5.3, 5.4, "tiny", db, "local")).toBe(true);
+ expect(item.data).toBe(
+ `<div class="table-container"><table></table></div><p>&lt;b&gt;</p><div class="table-container"><table></table></div>`
+ );
+ }));
+
describe.concurrent("[5.4] convert tiny to tiptap", () => {
const cases = [
{
@@ -106,6 +119,11 @@ describe.concurrent("[5.4] convert tiny to tiptap", () => {
from: `<p><img src="hello.jpg" /></p>`,
to: `<img src="hello.jpg">`
},
+ {
+ name: "keep text around images when moving them out of paragraphs",
+ from: `<p>before<img src="hello.jpg" />after</p>`,
+ to: `<p>before</p><img src="hello.jpg"><p>after</p>`
+ },
{
name: "replace [data-mce-bogus] elements",
from: `<p><br data-mce-bogus="" /></p><br data-mce-bogus="" />`,
diff --git a/packages/core/src/content-types/__tests__/tiptap.test.js b/packages/core/src/content-types/__tests__/tiptap.test.js
index 2b6923f18..13aa7ad99 100644
--- a/packages/core/src/content-types/__tests__/tiptap.test.js
+++ b/packages/core/src/content-types/__tests__/tiptap.test.js
@@ -92,3 +92,21 @@ for (const html in HTMLS) {
expect(tiptap.toTXT()).toMatchSnapshot(`html-to-txt-${html}.txt`);
});
}
+
+test("search ignores empty tokens", () => {
+ const tiptap = new Tiptap("<p>hello world</p>");
+ expect(tiptap.search("zzz ")).toBe(false);
+ expect(tiptap.search(" hello")).toBe(true);
+});
+
+test("list blocks include their nested lists' content", () => {
+ // the editor only assigns block ids to top-level blocks so nested lists have none
+ const tiptap = new Tiptap(
+ `<ul data-block-id="ul1"><li><p>parent</p><ul><li><p>child</p></li></ul></li><li><p>sibling</p></li></ul>`
+ );
+ const blocks = tiptap.extract("blocks").blocks;
+ expect(blocks).toHaveLength(1);
+ expect(blocks[0].content).toContain("parent");
+ expect(blocks[0].content).toContain("child");
+ expect(blocks[0].content).toContain("sibling");
+});
diff --git a/packages/core/src/content-types/tiptap.ts b/packages/core/src/content-types/tiptap.ts
index bcfbd3b4e..cd35f074d 100644
--- a/packages/core/src/content-types/tiptap.ts
+++ b/packages/core/src/content-types/tiptap.ts
@@ -102,7 +102,7 @@ export class Tiptap {
// }
search(query: string) {
- const tokens = query.toLowerCase().split(splitter);
+ const tokens = query.toLowerCase().split(splitter).filter(Boolean);
const lowercase = this.toTXT().toLowerCase();
return tokens.some((token) => lowercase.indexOf(token) > -1);
}
diff --git a/packages/core/src/migrations.ts b/packages/core/src/migrations.ts
index dd9734f42..8820f129f 100644
--- a/packages/core/src/migrations.ts
+++ b/packages/core/src/migrations.ts
@@ -657,7 +657,8 @@ function removeToxClassFromChecklist(html: string): string {
: document.body.innerHTML;
}
-const regex = /&lt;div class="table-container".*&lt;\/table&gt;&lt;\/div&gt;/gm;
+const regex =
+ /&lt;div class="table-container".*?&lt;\/table&gt;&lt;\/div&gt;/gm;
function decodeWrappedTableHtml(html: string) {
return html.replace(regex, (match) => {
const html = decodeHTML5(match);
@@ -669,6 +670,12 @@ const NEWLINE_REPLACEMENT_REGEX = /\n|<br>|<br\/>/gm;
const PREBLOCK_REGEX = /(<pre.*?>)(.*?)(<\/pre>)/gm;
const SPAN_REGEX = /<span class=.*?>(.*?)<\/span>/gm;
+function isEmpty(node: Node) {
+ return Array.from(node.childNodes).every(
+ (child) => child.nodeType === 3 && !child.textContent?.trim()
+ );
+}
+
export function tinyToTiptap(html: string) {
if (typeof html !== "string") return html;
@@ -703,7 +710,15 @@ export function tinyToTiptap(html: string) {
const images = document.querySelectorAll("p > img");
for (const image of images) {
- image.parentElement?.replaceWith(image.cloneNode());
+ const paragraph = image.parentElement;
+ if (!paragraph) continue;
+
+ // split the paragraph around the image so sibling content is kept
+ const after = paragraph.cloneNode(false);
+ while (image.nextSibling) after.appendChild(image.nextSibling);
+ paragraph.after(image);
+ if (!isEmpty(after)) image.after(after);
+ if (isEmpty(paragraph)) paragraph.remove();
}
const bogus = document.querySelectorAll("[data-mce-bogus]");
diff --git a/packages/core/src/utils/__tests__/content-block.test.ts b/packages/core/src/utils/__tests__/content-block.test.ts
index 8c332312d..e549c5809 100644
--- a/packages/core/src/utils/__tests__/content-block.test.ts
+++ b/packages/core/src/utils/__tests__/content-block.test.ts
@@ -39,36 +39,36 @@ describe("ContentBlock Utils", () => {
expect(links[0].text).toBe("link");
});
- it("should return an empty array if no internal links are present", () => {
+ it("should handle link text containing |", () => {
const block: ContentBlock = {
type: "someType",
id: "someId",
- content: "This is a test with no links"
+ content: "This is a test [[nn://note/123|a|b]]"
};
const links = extractInternalLinks(block);
- expect(links).toHaveLength(0);
+ expect(links).toHaveLength(1);
+ expect(links[0].id).toBe("123");
+ expect(links[0].text).toBe("a|b");
});
- it("should skip links with no noteId", () => {
+ it("should return an empty array if no internal links are present", () => {
const block: ContentBlock = {
type: "someType",
id: "someId",
- content: "This is a test [[nn://note/|link]] with undefined URL"
+ content: "This is a test with no links"
};
const links = extractInternalLinks(block);
expect(links).toHaveLength(0);
});
- it("should extract internal links with '|' in id", () => {
+ it("should skip links with no noteId", () => {
const block: ContentBlock = {
type: "someType",
id: "someId",
- content: "This is a test [[nn://note/myid|ofmyid|actualtext]]"
+ content: "This is a test [[nn://note/|link]] with undefined URL"
};
const links = extractInternalLinks(block);
- expect(links).toHaveLength(1);
- expect(links[0].id).toBe("myid|ofmyid");
- expect(links[0].text).toBe("actualtext");
+ expect(links).toHaveLength(0);
});
});
diff --git a/packages/core/src/utils/__tests__/fuzzy.test.ts b/packages/core/src/utils/__tests__/fuzzy.test.ts
index 5b576ffe3..d5c544540 100644
--- a/packages/core/src/utils/__tests__/fuzzy.test.ts
+++ b/packages/core/src/utils/__tests__/fuzzy.test.ts
@@ -130,3 +130,10 @@ describe("lookup.fuzzy", () => {
});
});
});
+
+test("sanitized query is tried when the raw query has no matches", () => {
+ const items = [{ id: "1", title: "hello world" }];
+ expect(fuzzy("hello-world", items, (i) => i.id, { title: 1 })).toStrictEqual(
+ items
+ );
+});
diff --git a/packages/core/src/utils/__tests__/html-rewriter.test.ts b/packages/core/src/utils/__tests__/html-rewriter.test.ts
new file mode 100644
index 000000000..d228f74d2
--- /dev/null
+++ b/packages/core/src/utils/__tests__/html-rewriter.test.ts
@@ -0,0 +1,37 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { test, expect } from "vitest";
+import { HTMLRewriter } from "../html-rewriter.js";
+
+test("implied close of a non-void element doesn't write /> into text", () => {
+ expect(new HTMLRewriter().transform("<p>a<p>b")).toBe("<p>a</p><p>b</p>");
+});
+
+test("void and self-closing elements are still self-closed", () => {
+ expect(new HTMLRewriter().transform('<p>a<img src="x"><br/>b</p>')).toBe(
+ '<p>a<img src="x"/><br/>b</p>'
+ );
+});
+
+test("attribute values containing quotes are escaped", () => {
+ expect(new HTMLRewriter().transform(`<p title='say "hi"'>a</p>`)).toBe(
+ '<p title="say &quot;hi&quot;">a</p>'
+ );
+});
diff --git a/packages/core/src/utils/__tests__/templates.test.ts b/packages/core/src/utils/__tests__/templates.test.ts
new file mode 100644
index 000000000..5887691b1
--- /dev/null
+++ b/packages/core/src/utils/__tests__/templates.test.ts
@@ -0,0 +1,34 @@
+/*
+This file is part of the Notesnook project (https://notesnook.com/)
+
+Copyright (C) 2023 Streetwriters (Private) Limited
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+*/
+
+import { test, expect } from "vitest";
+import { templateWithFrontmatter } from "../templates/md.js";
+
+test("frontmatter quotes tags and color", () => {
+ const md = templateWithFrontmatter({
+ title: "Note",
+ content: "",
+ dateCreated: 0,
+ dateEdited: 0,
+ color: "red: #f00",
+ tags: ["a: b", "#c"]
+ } as any);
+ expect(md).toContain('color: "red: #f00"');
+ expect(md).toContain('tags: "a: b, #c"');
+});
diff --git a/packages/core/src/utils/__tests__/title-format.test.js b/packages/core/src/utils/__tests__/title-format.test.js
index d34df0c63..abfc1dd6f 100644
--- a/packages/core/src/utils/__tests__/title-format.test.js
+++ b/packages/core/src/utils/__tests__/title-format.test.js
@@ -78,3 +78,16 @@ describe("day format", () => {
).toBe("Note Wed");
});
});
+
+test("$headline$ with replacement patterns is inserted literally", () => {
+ expect(
+ formatTitle(
+ "Note $headline$",
+ "[DD-MM-YYYY]",
+ "[hh:mm]",
+ "short",
+ "costs $& more $$ $'",
+ 0
+ )
+ ).toBe("Note costs $& more $$ $'");
+});
diff --git a/packages/core/src/utils/content-block.ts b/packages/core/src/utils/content-block.ts
index d77c97c65..fee3617a0 100644
--- a/packages/core/src/utils/content-block.ts
+++ b/packages/core/src/utils/content-block.ts
@@ -27,8 +27,8 @@ export function extractInternalLinks(block: ContentBlock) {
const links: InternalLinkWithOffset[] = [];
for (const match of matches) {
if (match.index === undefined) continue;
- const url = match[1].slice(0, match[1].lastIndexOf("|"));
- const text = match[1].slice(match[1].lastIndexOf("|") + 1);
+ const url = match[1].slice(0, match[1].indexOf("|"));
+ const text = match[1].slice(match[1].indexOf("|") + 1);
const link = parseInternalLink(url);
if (!link) continue;
links.push({
diff --git a/packages/core/src/utils/fuzzy.ts b/packages/core/src/utils/fuzzy.ts
index d51b25c24..044f7f3d2 100644
--- a/packages/core/src/utils/fuzzy.ts
+++ b/packages/core/src/utils/fuzzy.ts
@@ -32,7 +32,6 @@ export function fuzzy<T>(
} = {}
): T[] {
const results = fuzzyMatch(query, items, getIdentifier, fields, options);
- if (results.size === 0) return [];
// if the query contains spaces & other non-alphanumeric characters, then we
// should search again with a sanitized query to catch more matches.
diff --git a/packages/core/src/utils/html-rewriter.ts b/packages/core/src/utils/html-rewriter.ts
index cfea11976..0d88f366b 100644
--- a/packages/core/src/utils/html-rewriter.ts
+++ b/packages/core/src/utils/html-rewriter.ts
@@ -86,7 +86,7 @@ export class HTMLRewriter {
if (attr) {
for (const key in attr) {
if (!key) continue;
- this.write(` ${key}="${attr[key]}"`);
+ this.write(` ${key}="${attr[key].replace(/"/g, "&quot;")}"`);
}
}
this.currentTag = name;
@@ -104,9 +104,12 @@ export class HTMLRewriter {
if (this.skippedTags.pop()) return;
- if (!isImplied) this.closeTag();
+ // an implied close right after the open tag is a void or
+ // self-closing element; otherwise the element had content.
+ const selfClose = isImplied && this.currentTag !== null;
+ if (!selfClose) this.closeTag();
- this.write(isImplied ? "/>" : `</${name}>`);
+ this.write(selfClose ? "/>" : `</${name}>`);
if (this.currentTag) {
this.currentTag = null;
diff --git a/packages/core/src/utils/templates/html/index.ts b/packages/core/src/utils/templates/html/index.ts
index 172d4252c..fa9922c63 100644
--- a/packages/core/src/utils/templates/html/index.ts
+++ b/packages/core/src/utils/templates/html/index.ts
@@ -97,6 +97,9 @@ async function preprocessHTML(templateData: TemplateData) {
const codeblocks = doc.querySelectorAll("pre > code");
if (codeblocks.length) {
+ // stop Prism from auto-highlighting the host document on import
+ const globalScope = globalThis as { Prism?: { manual?: boolean } };
+ if (!globalScope.Prism) globalScope.Prism = { manual: true };
const { default: prismjs } = await import("prismjs");
// const { loadLanguage } = await import("./languages/index.js");
(prismjs as any).register = (syntax: (syntax: any) => void) => {
diff --git a/packages/core/src/utils/templates/md.ts b/packages/core/src/utils/templates/md.ts
index 5b096a6c5..feb54bf5e 100644
--- a/packages/core/src/utils/templates/md.ts
+++ b/packages/core/src/utils/templates/md.ts
@@ -41,7 +41,7 @@ function buildFrontmatter(data: TemplateData) {
if (data.pinned) lines.push(`pinned: ${data.pinned}`);
if (data.favorite) lines.push(`favorite: ${data.favorite}`);
if (data.archived) lines.push(`archived: ${data.archived}`);
- if (data.color) lines.push(`color: ${data.color}`);
- if (data.tags) lines.push(`tags: ${data.tags.join(", ")}`);
+ if (data.color) lines.push(`color: ${JSON.stringify(data.color)}`);
+ if (data.tags) lines.push(`tags: ${JSON.stringify(data.tags.join(", "))}`);
return lines.join("\n");
}
diff --git a/packages/core/src/utils/title-format.ts b/packages/core/src/utils/title-format.ts
index ce3e41a69..a3b277b0c 100644
--- a/packages/core/src/utils/title-format.ts
+++ b/packages/core/src/utils/title-format.ts
@@ -64,7 +64,7 @@ export function formatTitle(
.replace(DATE_REGEX, date)
.replace(TIME_REGEX, time)
.replace(DAY_REGEX, day)
- .replace(HEADLINE_REGEX, headline || "")
+ .replace(HEADLINE_REGEX, () => headline || "")
.replace(TIMESTAMP_REGEX, timestamp)
.replace(TIMESTAMP_Z_REGEX, timestampWithTimeZone)
.replace(COUNT_REGEX, `${totalNotes + 1}`);