Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

fix(editor): support backslash-escaping literal #tags

boojack5f7e038aa76a
diff --git a/web/src/components/MemoEditor/Editor/Tag.ts b/web/src/components/MemoEditor/Editor/Tag.ts
index 12734d037..478cabc91 100644
--- a/web/src/components/MemoEditor/Editor/Tag.ts
+++ b/web/src/components/MemoEditor/Editor/Tag.ts
@@ -1,93 +1,19 @@
import type { MarkdownToken } from "@tiptap/core";
import { InputRule, Mark, mergeAttributes } from "@tiptap/core";
-import type { TokenizerThis, Tokens } from "marked";
-import { marked } from "marked";
import { tagStyles } from "@/lib/markdownStyles";
-import { MAX_TAG_LENGTH, TAG_CHAR_CLASS } from "@/utils/tag-grammar";
+import { TAG_RUN } from "@/utils/tag-grammar";
// Default tag pill, shared with the read-only view (MemoContent/Tag.tsx).
// Computed once — renderHTML runs on every view update.
const TAG_CLASS = `${tagStyles.base} ${tagStyles.defaultColor}`;
-// Built from the shared tag grammar (@/utils/tag-grammar) so the editor's
-// tokenizer/input rule can't drift from the read-only renderer's lexer
-// (web/src/utils/remark-plugins/remark-tag.ts).
-const TAG_INPUT_RULE = new RegExp(`(?:^|\\s)#(${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}})\\s$`, "u");
-const TAG_TOKEN_RULE = new RegExp(`^#(${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}})`, "u");
-// Tests the REMAINDER of the source (not a single code unit) so astral-plane
-// tag characters (emoji et al.) are seen whole, not as lone surrogates.
-const TAG_CHAR_AHEAD = new RegExp(`^${TAG_CHAR_CLASS}`, "u");
-
-/**
- * Tag tokenizer, registered DIRECTLY on the global marked singleton instead of
- * through `markdownTokenizer`. Two reasons:
- *
- * 1. `@tiptap/markdown`'s MarkdownManager wraps `markdownTokenizer.tokenize`
- * in `tokenizer(src, tokens) { ... tokenize(src, tokens, helper) }` — the
- * wrapper receives marked's TokenizerThis (with `lexer.state.inLink`) but
- * does NOT forward it, so a manager-registered tokenizer can never know it
- * is inside a link label. Registered natively, marked invokes us with
- * `this.lexer` bound and we can decline inside `[label](url)` the same way
- * remark-tag skips link nodes.
- * 2. The manager defaults to the global `marked` export (`markedInstance =
- * options?.marked ?? marked`) and `web` resolves the exact same marked
- * module instance as `@tiptap/markdown` does, so this registration is
- * visible to every lexer the manager creates. Module scope + idempotent:
- * registered exactly once per page load (the manager's own per-Editor
- * `marked.use` calls are the accumulation hazard documented in
- * markdownCodec.ts; this adds a single registration, ever).
- *
- * The Tag mark below still declares `markdownTokenName: "memoTag"` +
- * `parseMarkdown`, which is all the manager needs to route the token.
- */
-function tokenizeTag(this: TokenizerThis, src: string): Tokens.Generic | undefined {
- // remark-tag skips link nodes entirely; marked sets `state.inLink` while
- // tokenizing link/reflink labels, so declining here keeps `[see #x](url)`
- // a plain link label instead of tearing the tag out of it.
- if (this.lexer?.state?.inLink) {
- return undefined;
- }
- const match = TAG_TOKEN_RULE.exec(src);
- if (!match) {
- return undefined;
- }
- const rest = src.slice(match[0].length);
- // `#a#b`: decline when the run is directly followed by another `#`, so the
- // FIRST run stays plain text. NOT full remark parity — remark-tag treats
- // `#a#b` as two tags and `##x` as all-text, while here `#a#b` becomes text
- // "#a" + tag "b" and `##x` becomes text "#" + tag "x". The divergence is
- // visual-only in the editor: both shapes serialize back byte-identically
- // (the surrounding text nodes re-emit their literal characters).
- if (rest.startsWith("#")) {
- return undefined;
- }
- // Runs longer than 100 tag characters are not tags at all in remark-tag
- // (the whole run stays plain text) — decline instead of splitting the run
- // into a 100-char tag plus leftover text.
- if (TAG_CHAR_AHEAD.test(rest)) {
- return undefined;
- }
- return { type: "memoTag", raw: match[0], text: match[0], tag: match[1] };
-}
-
-let tagTokenizerRegistered = false;
-function registerTagTokenizer() {
- if (tagTokenizerRegistered) {
- return;
- }
- tagTokenizerRegistered = true;
- marked.use({
- extensions: [
- {
- name: "memoTag",
- level: "inline",
- start: (src: string) => src.indexOf("#"),
- tokenizer: tokenizeTag,
- },
- ],
- });
-}
-registerTagTokenizer();
+// Built from the shared TAG_RUN (@/utils/tag-grammar) so the editor's input
+// rule and tokenizer can't drift from the serialize-escape or the read-only
+// renderer's lexer (web/src/utils/remark-plugins/remark-tag.ts). The capped-run
+// lookahead in TAG_RUN also makes an over-long run decline to match — no
+// separate length check needed.
+const TAG_INPUT_RULE = new RegExp(`(?:^|\\s)#(${TAG_RUN})\\s$`, "u");
+const TAG_TOKEN_RULE = new RegExp(`^#(${TAG_RUN})`, "u");
/**
* Mark for memos `#tags`: styled in the editor, serialized back to `#tag`
@@ -99,8 +25,20 @@ registerTagTokenizer();
* `applyMarkToContent` only attaches enclosing marks to text nodes, so an
* atom inside bold silently drops the bold delimiters.
*
- * Parsed live while typing (input rule) and from markdown (the native marked
- * tokenizer above).
+ * Parsed live while typing (the input rule) and from markdown (the
+ * `markdownTokenizer` below — the canonical @tiptap/markdown extension point,
+ * https://tiptap.dev/docs/editor/markdown/advanced-usage/custom-tokenizer).
+ *
+ * Two consequences of going through the manager (vs. registering on `marked`
+ * directly) are handled in tagMarkdown.ts:
+ * - The tokenizer can't see whether it sits inside a link label — the manager
+ * forwards only `{ inlineTokens, blockTokens }`, not the lexer's `inLink`
+ * state — so tag-in-link skipping is a tree pass there, mirroring how the
+ * read-only renderer (remark-tag) skips link nodes.
+ * - The manager re-registers this tokenizer onto the global `marked` on every
+ * Editor construction (the accumulation noted in markdownCodec.ts). The
+ * impact is sub-millisecond for memo-sized content and the test codec is a
+ * singleton; accepted as the cost of the canonical API.
*/
export const Tag = Mark.create({
name: "tag",
@@ -154,6 +92,26 @@ export const Tag = Mark.create({
];
},
+ markdownTokenizer: {
+ name: "memoTag",
+ level: "inline",
+ start: (src: string) => src.indexOf("#"),
+ tokenize: (src: string) => {
+ const match = TAG_TOKEN_RULE.exec(src);
+ if (!match) {
+ return undefined;
+ }
+ // `#a#b`: decline when the run is directly followed by another `#`, so the
+ // FIRST run stays plain text "#a" + tag "b" (`##x` likewise → text "#" +
+ // tag "x"). The serialize-escape then escapes the tag-shaped literal "#a",
+ // so `#a#b` round-trips as `\#a#b` — doc-equivalent and stable thereafter.
+ // (An over-long run already declines: TAG_RUN's lookahead fails to match.)
+ if (src.slice(match[0].length).startsWith("#")) {
+ return undefined;
+ }
+ return { type: "memoTag", raw: match[0], tag: match[1] };
+ },
+ },
markdownTokenName: "memoTag",
parseMarkdown: (token, helpers) => {
const t = token as MarkdownToken & { tag?: string };
diff --git a/web/src/components/MemoEditor/Editor/extensions.ts b/web/src/components/MemoEditor/Editor/extensions.ts
index ff8b32e3f..fa941a08d 100644
--- a/web/src/components/MemoEditor/Editor/extensions.ts
+++ b/web/src/components/MemoEditor/Editor/extensions.ts
@@ -1,11 +1,11 @@
import { type AnyExtension, mergeAttributes } from "@tiptap/core";
import { Heading } from "@tiptap/extension-heading";
import { TaskItem, TaskList } from "@tiptap/extension-list";
-import { Markdown } from "@tiptap/markdown";
import StarterKit from "@tiptap/starter-kit";
import { type HeadingLevel, headingClass, markdownStyles } from "@/lib/markdownStyles";
import { preservedExtensions } from "./PreservedBlock";
import { Tag } from "./Tag";
+import { TagAwareMarkdown } from "./tagMarkdown";
/**
* StarterKit's Heading is bundled and cannot vary classes by level via static
@@ -46,7 +46,7 @@ export function buildExtensions(): AnyExtension[] {
StyledHeading.configure({ levels: [1, 2, 3, 4, 5, 6] }),
TaskList,
TaskItem.configure({ nested: true }),
- Markdown,
+ TagAwareMarkdown,
...preservedExtensions,
Tag,
];
diff --git a/web/src/components/MemoEditor/Editor/tagMarkdown.ts b/web/src/components/MemoEditor/Editor/tagMarkdown.ts
new file mode 100644
index 000000000..eedfe4634
--- /dev/null
+++ b/web/src/components/MemoEditor/Editor/tagMarkdown.ts
@@ -0,0 +1,113 @@
+import type { JSONContent } from "@tiptap/core";
+import { Markdown } from "@tiptap/markdown";
+import type { Schema } from "@tiptap/pm/model";
+import { TAG_RUN } from "@/utils/tag-grammar";
+
+// A `#` begins a memo tag when a capped TAG_RUN follows it — the same grammar
+// the editor tokenizer (Tag.ts) matches, so escape and parse can't disagree.
+// The lookahead matches only the `#`, leaving the run itself untouched.
+const TAG_HASH = new RegExp(`#(?=${TAG_RUN})`, "gu");
+
+/**
+ * Backslash-escape a `#` that would otherwise re-parse into a `#tag`.
+ *
+ * memos layers hashtag syntax on top of CommonMark, so — exactly as
+ * prosemirror-markdown escapes inline syntax characters like `*` and `_` —
+ * plain text that looks like a tag must be escaped on serialize, or it silently
+ * becomes a tag on the next parse.
+ */
+export function escapeTagHashes(text: string): string {
+ return text.replace(TAG_HASH, "\\#");
+}
+
+type JsonMark = string | { type?: string };
+type JsonNode = { type?: string; marks?: JsonMark[] };
+type PatchableManager = {
+ encodeTextForMarkdown?: (text: string, node: JsonNode, parentNode?: JsonNode) => string;
+ parse?: (markdown: string) => JSONContent;
+};
+
+const markName = (mark: JsonMark): string => (typeof mark === "string" ? mark : (mark.type ?? ""));
+const hasMark = (node: JsonNode, name: string): boolean => (node.marks ?? []).some((mark) => markName(mark) === name);
+
+/**
+ * A `#tag` only forms in plain inline prose. The tokenizer declines inside
+ * links, and `code: true` marks/nodes (the Tag mark itself, inline code,
+ * preserved spans/blocks, code blocks) serialize verbatim — so escaping a `#`
+ * in any of those contexts would be wrong (and would corrupt the byte-identical
+ * round-trips those constructs rely on).
+ */
+function escapesTagsHere(schema: Schema, node: JsonNode, parentNode?: JsonNode): boolean {
+ for (const mark of node.marks ?? []) {
+ const name = markName(mark);
+ if (name === "link") {
+ return false;
+ }
+ if (schema.marks[name]?.spec.code) {
+ return false;
+ }
+ }
+ const parentType = parentNode?.type;
+ if (parentType && schema.nodes[parentType]?.spec.code) {
+ return false;
+ }
+ return true;
+}
+
+/**
+ * Strip the `tag` mark from any text that also carries a `link` mark.
+ *
+ * The `markdownTokenizer` in Tag.ts cannot tell it is inside a link label — the
+ * manager forwards only `{ inlineTokens, blockTokens }`, never the lexer's
+ * `inLink` state — so `[see #x](url)` parses with `#x` tagged. We undo that as a
+ * tree pass, the same way the read-only renderer (remark-tag) skips link nodes,
+ * keeping the editor and renderer in agreement: a `#` in a link label is link
+ * text, never a tag pill.
+ */
+function stripTagInsideLinks(node: JSONContent): JSONContent {
+ if (Array.isArray(node.marks) && hasMark(node, "link")) {
+ node.marks = node.marks.filter((mark) => markName(mark) !== "tag");
+ }
+ node.content?.forEach(stripTagInsideLinks);
+ return node;
+}
+
+/**
+ * The `markdownTokenizer` in Tag.ts handles `#tag` the canonical
+ * @tiptap/markdown way, but two memos-specific concerns sit above the tokenizer
+ * and have no public hook, so we compose them onto the manager here:
+ *
+ * - serialize: escape a tag-shaped `#` in plain prose ({@link escapeTagHashes})
+ * so a literal `#NAS` can't silently re-parse into a tag;
+ * - parse: skip tags inside link labels ({@link stripTagInsideLinks}).
+ *
+ * Escapes stay a purely lexical concern — the document model carries no
+ * "escaped tag" node or mark. A literal `#NAS` is just text: parsing strips a
+ * leading `\` (marked's built-in escape) and serializing adds it back, the same
+ * way TipTap handles every other escapable character.
+ */
+export const TagAwareMarkdown = Markdown.extend({
+ onBeforeCreate(props) {
+ this.parent?.(props);
+ const { editor } = this;
+ const manager = editor.markdown as unknown as PatchableManager | undefined;
+ if (!manager) {
+ return;
+ }
+
+ const encode = manager.encodeTextForMarkdown;
+ if (typeof encode === "function") {
+ const bound = encode.bind(manager);
+ manager.encodeTextForMarkdown = (text, node, parentNode) => {
+ const encoded = bound(text, node, parentNode);
+ return escapesTagsHere(editor.schema, node, parentNode) ? escapeTagHashes(encoded) : encoded;
+ };
+ }
+
+ const parse = manager.parse;
+ if (typeof parse === "function") {
+ const bound = parse.bind(manager);
+ manager.parse = (markdown) => stripTagInsideLinks(bound(markdown));
+ }
+ },
+});
diff --git a/web/src/components/MemoEditor/README.md b/web/src/components/MemoEditor/README.md
index 2b2a40ad6..afbd46a57 100644
--- a/web/src/components/MemoEditor/README.md
+++ b/web/src/components/MemoEditor/README.md
@@ -45,7 +45,8 @@ MemoEditor/
│ ├── extensions.ts # Canonical schema-relevant extension set (shared with codec)
│ ├── markdownCodec.ts # Headless parse/serialize helpers (singleton editor)
│ ├── PreservedBlock.ts # Byte-for-byte preservation of tables, math, raw HTML
-│ ├── Tag.ts # Memos #tag mark
+│ ├── Tag.ts # Memos #tag mark (markdownTokenizer + input rule)
+│ ├── tagMarkdown.ts # Tag-aware Markdown: # escape on serialize, link-skip on parse
│ ├── TagSuggestion.ts # # popup for WYSIWYG mode
│ └── suggestionMenu.tsx # Shared suggestion popup renderer (used by TagSuggestion)
├── Toolbar/ # Toolbar sub-components (InsertMenu, VisibilitySelector)
@@ -85,7 +86,17 @@ Mode switching is a markdown handoff: because both editors write into `state.con
`PreservedBlock.ts` handles syntax the WYSIWYG editor does not model richly: tables, `$$math$$`, and raw HTML are captured at parse time with their raw markdown source, shown as editable monospace literal text, and re-emitted byte-for-byte on serialize.
-`Tag.ts` models memos `#tags` as a `code: true` text mark, letting tags round-trip byte-identically even inside bold or heading spans.
+`Tag.ts` models memos `#tags` as a `code: true` text mark, letting tags round-trip byte-identically even inside bold or heading spans. Its `#tag` lexing is a `markdownTokenizer` (the canonical `@tiptap/markdown` extension point); `tagMarkdown.ts` adds the two things that tokenizer can't do — on serialize it backslash-escapes a literal `#` that would otherwise re-parse into a tag (escapes are lexical, so there is no "escaped tag" node), and on parse it strips the tag mark from text inside link labels. The `#tag` grammar itself lives once in `utils/tag-grammar.ts` (`TAG_RUN`), shared by the tokenizer, the serialize-escape, and the read-only `remark-tag` renderer so they can't drift.
+
+### Why the markdown manager is worked around in several places
+
+`@tiptap/markdown` (3.26.0) exposes no public, per-instance hook for custom tokenizers or for text escaping, and it registers each extension's tokenizer onto the **global** `marked` singleton on every `new Editor()` — registrations it never removes. That one limitation is the reason for three otherwise-surprising choices, each documented in detail at its call site:
+
+- **`markdownCodec.ts` keeps a single editor instance** — re-creating editors would leak tokenizer registrations onto global `marked` and measurably degrade parse time.
+- **`PreservedBlock.ts` registers its tokenizers on `marked` once at module scope** (idempotent) instead of via the per-extension `markdownTokenizer`, to avoid that per-construction accumulation.
+- **`tagMarkdown.ts` composes onto the manager in `onBeforeCreate`** (escape, link-skip) because the relevant manager methods are `private` with no public seam.
+
+`Tag.ts` deliberately uses the canonical `markdownTokenizer` API and accepts the per-construction re-registration as its cost; `PreservedBlock.ts` refuses it for its seven tokenizers. The asymmetry is intentional — collapse both onto one path if upstream ever ships a public per-instance tokenizer hook.
### Suggestions
diff --git a/web/src/utils/remark-plugins/remark-tag.ts b/web/src/utils/remark-plugins/remark-tag.ts
index e21847189..0271f1771 100644
--- a/web/src/utils/remark-plugins/remark-tag.ts
+++ b/web/src/utils/remark-plugins/remark-tag.ts
@@ -1,18 +1,64 @@
import type { Root, Text } from "mdast";
-import type { Node as UnistNode } from "unist";
+import type { Position, Node as UnistNode } from "unist";
import type { TagNode, TagNodeData } from "@/types/markdown";
import { isTagChar, MAX_TAG_LENGTH } from "@/utils/tag-grammar";
-function parseTagsFromText(text: string): Array<{ type: "text"; value: string } | { type: "tag"; value: string }> {
- const segments: Array<{ type: "text"; value: string } | { type: "tag"; value: string }> = [];
+type Segment = { type: "text"; value: string } | { type: "tag"; value: string };
- const chars = [...text];
+// CommonMark "ASCII punctuation": the only characters a leading backslash can
+// escape. A backslash before anything else is a literal backslash.
+function isAsciiPunctuation(char: string): boolean {
+ if (char.length !== 1) {
+ return false;
+ }
+ const code = char.charCodeAt(0);
+ return (
+ (code >= 0x21 && code <= 0x2f) || // ! " # $ % & ' ( ) * + , - . /
+ (code >= 0x3a && code <= 0x40) || // : ; < = > ? @
+ (code >= 0x5b && code <= 0x60) || // [ \ ] ^ _ `
+ (code >= 0x7b && code <= 0x7e) // { | } ~
+ );
+}
+
+/**
+ * Apply CommonMark backslash-unescaping to a raw source slice, tracking which
+ * resulting characters came from an escape. `\#` yields a `#` flagged escaped,
+ * so the tag lexer can tell a deliberately-escaped hash from a real tag.
+ * Returns code points (not UTF-16 units) so astral characters stay intact.
+ */
+function unescapeBackslashes(source: string): { chars: string[]; escaped: boolean[] } {
+ const codePoints = [...source];
+ const chars: string[] = [];
+ const escaped: boolean[] = [];
+
+ for (let i = 0; i < codePoints.length; i++) {
+ if (codePoints[i] === "\\" && i + 1 < codePoints.length && isAsciiPunctuation(codePoints[i + 1])) {
+ chars.push(codePoints[i + 1]);
+ escaped.push(true);
+ i++;
+ continue;
+ }
+ chars.push(codePoints[i]);
+ escaped.push(false);
+ }
+
+ return { chars, escaped };
+}
+
+/**
+ * Split a run of characters into text/tag segments. `escaped[i]` marks a
+ * character that came from a backslash escape: an escaped `#` can never start a
+ * tag (so `\#NAS` stays literal text), matching the backend goldmark parser and
+ * the editor's marked tokenizer, both of which honor the escape natively.
+ */
+function parseSegments(chars: string[], escaped: boolean[]): Segment[] {
+ const segments: Segment[] = [];
let i = 0;
while (i < chars.length) {
- if (chars[i] === "#" && i + 1 < chars.length && isTagChar(chars[i + 1])) {
+ if (chars[i] === "#" && !escaped[i] && i + 1 < chars.length && isTagChar(chars[i + 1])) {
const prevChar = i > 0 ? chars[i - 1] : "";
- const nextChar = i + 1 < chars.length ? chars[i + 1] : "";
+ const nextChar = chars[i + 1];
if (prevChar === "#" || nextChar === "#" || nextChar === " ") {
segments.push({ type: "text", value: chars[i] });
@@ -35,8 +81,9 @@ function parseTagsFromText(text: string): Array<{ type: "text"; value: string }
}
}
+ // Consume a plain-text run up to the next tag-eligible (non-escaped) hash.
let j = i + 1;
- while (j < chars.length && chars[j] !== "#") {
+ while (j < chars.length && !(chars[j] === "#" && !escaped[j])) {
j++;
}
segments.push({ type: "text", value: chars.slice(i, j).join("") });
@@ -46,6 +93,34 @@ function parseTagsFromText(text: string): Array<{ type: "text"; value: string }
return segments;
}
+/**
+ * Segment a text node, preferring the original source slice (where escapes are
+ * still visible) over the post-escape `value`. The source-derived parse is only
+ * trusted when it reconstructs `value` byte-for-byte; otherwise — entity
+ * references, missing positions, upstream rewrites — we fall back to the
+ * value-based parse, which is exactly the pre-escape-support behavior. This
+ * makes escape support strictly additive: it never changes a node it can't
+ * faithfully account for.
+ */
+function segmentsForTextNode(value: string, position: Position | undefined, source: string): Segment[] {
+ const startOffset = position?.start?.offset;
+ const endOffset = position?.end?.offset;
+
+ if (source && startOffset != null && endOffset != null) {
+ const slice = source.slice(startOffset, endOffset);
+ const { chars, escaped } = unescapeBackslashes(slice);
+ if (chars.join("") === value) {
+ return parseSegments(chars, escaped);
+ }
+ }
+
+ const chars = [...value];
+ return parseSegments(
+ chars,
+ chars.map(() => false),
+ );
+}
+
function createTagNode(tagValue: string): TagNode {
const data: TagNodeData = {
hName: "span",
@@ -73,14 +148,14 @@ function isLinkNode(node: UnistNode): boolean {
return node.type === "link" || node.type === "linkReference";
}
-function transformTagTextNodes(parent: ParentNode, insideLink: boolean): void {
+function transformTagTextNodes(parent: ParentNode, insideLink: boolean, source: string): void {
for (let index = 0; index < parent.children.length; index++) {
const child = parent.children[index];
const childInsideLink = insideLink || isLinkNode(child);
if (child.type === "text" && !childInsideLink) {
const textNode = child as Text;
- const segments = parseTagsFromText(textNode.value);
+ const segments = segmentsForTextNode(textNode.value, textNode.position, source);
if (segments.every((seg) => seg.type === "text")) {
continue;
@@ -102,13 +177,16 @@ function transformTagTextNodes(parent: ParentNode, insideLink: boolean): void {
}
if (isParentNode(child)) {
- transformTagTextNodes(child, childInsideLink);
+ transformTagTextNodes(child, childInsideLink, source);
}
}
}
+type VFileLike = { value?: string | Uint8Array };
+
export const remarkTag = () => {
- return (tree: Root) => {
- transformTagTextNodes(tree as ParentNode, false);
+ return (tree: Root, file: VFileLike) => {
+ const source = typeof file?.value === "string" ? file.value : "";
+ transformTagTextNodes(tree as ParentNode, false, source);
};
};
diff --git a/web/src/utils/tag-grammar.ts b/web/src/utils/tag-grammar.ts
index b35d16f80..dcd819742 100644
--- a/web/src/utils/tag-grammar.ts
+++ b/web/src/utils/tag-grammar.ts
@@ -1,7 +1,8 @@
/**
* The single source of truth for memos' `#tag` lexing grammar, shared by the
- * editor tokenizer (components/MemoEditor/Editor/Tag.ts) and the read-only
- * renderer (utils/remark-plugins/remark-tag.ts) so the two can't drift.
+ * editor tokenizer + serialize-escape (components/MemoEditor/Editor/Tag.ts and
+ * tagMarkdown.ts) and the read-only renderer (utils/remark-plugins/remark-tag.ts)
+ * so they can't drift.
*
* A tag character is any Unicode letter, number, or symbol, plus `_ - / &`.
* A tag run is capped at MAX_TAG_LENGTH characters.
@@ -10,6 +11,18 @@ export const TAG_CHAR_CLASS = "[\\p{L}\\p{N}\\p{S}_\\-/&]";
export const MAX_TAG_LENGTH = 100;
+/**
+ * Regex source for a *capped* tag run: 1..MAX_TAG_LENGTH tag characters,
+ * refusing to match when a (MAX+1)-th tag character would follow (an over-long
+ * run is not a tag). Embed with the `u` flag.
+ *
+ * The two halves of the round-trip build on this same source so they can't
+ * disagree on what counts as a tag: the editor's input rule + tokenizer match
+ * `#(${TAG_RUN})`, and the serialize-escape (tagMarkdown.ts) escapes a `#`
+ * followed by `(?=${TAG_RUN})`.
+ */
+export const TAG_RUN = `${TAG_CHAR_CLASS}{1,${MAX_TAG_LENGTH}}(?!${TAG_CHAR_CLASS})`;
+
// Matches exactly one tag character. The `u` flag makes the class match whole
// code points, so astral-plane symbols (emoji et al.) are tested intact rather
// than as lone surrogates.
diff --git a/web/tests/editor-controller.test.tsx b/web/tests/editor-controller.test.tsx
index 4769854b6..bbe1ca616 100644
--- a/web/tests/editor-controller.test.tsx
+++ b/web/tests/editor-controller.test.tsx
@@ -105,6 +105,43 @@ describe("tag suggestion insertion", () => {
});
});
+describe("tag escaping", () => {
+ // Drives the ProseMirror input-rules plugin the way real typing does:
+ // someProp("handleTextInput", …) is exactly what the view calls on keystroke.
+ function typeText(editor: EditorInstance, text: string) {
+ const { from } = editor.state.selection;
+ editor.view.someProp("handleTextInput", (handler) => handler(editor.view, from, from, text));
+ }
+
+ it("undoing the tag autoformat leaves a durable literal #tag, not a pill", () => {
+ const { ref, editor } = setup("");
+ act(() => {
+ editor.view.dispatch(editor.state.tr.insertText("#NAS", 1));
+ });
+ act(() => typeText(editor, " ")); // tag input rule fires → pill
+ act(() => {
+ editor.commands.undoInputRule(); // the standard "I didn't want that" gesture
+ });
+
+ // Back to plain text — and the serializer escapes it so it can't re-tag.
+ expect(ref.current?.getMarkdown()?.trim()).toBe("\\#NAS");
+ const para = editor.getJSON().content?.[0];
+ expect(para?.content?.some((n) => (n.marks ?? []).some((m) => m.type === "tag"))).toBeFalsy();
+ });
+
+ it("a plain #tag still becomes a tag", () => {
+ const { ref, editor } = setup("");
+ act(() => {
+ editor.view.dispatch(editor.state.tr.insertText("#work", 1));
+ });
+ act(() => typeText(editor, " "));
+
+ expect(ref.current?.getMarkdown()?.trim()).toBe("#work");
+ const para = editor.getJSON().content?.[0];
+ expect(para?.content?.[0]).toMatchObject({ text: "#work", marks: [{ type: "tag" }] });
+ });
+});
+
describe("external content sync", () => {
it("a trim-equal echo of the editor's own output does not reset the document", () => {
const { ref, rerender } = setupRerenderable("hello");
diff --git a/web/tests/editor-tag.test.ts b/web/tests/editor-tag.test.ts
index e23e1cd7f..5b7e93928 100644
--- a/web/tests/editor-tag.test.ts
+++ b/web/tests/editor-tag.test.ts
@@ -96,9 +96,13 @@ describe("Tag mark", () => {
expect(roundTripMarkdown(input).trim()).toBe(input);
});
- it("round-trips ##x and #a#b byte-identically (visual divergence documented in Tag.ts)", () => {
+ it("round-trips ##x unchanged and normalizes #a#b to \\#a#b (conservative # escaping)", () => {
+ // "##x" → text "#" + tag "x"; the lone "#" is not tag-shaped, so untouched.
expect(roundTripMarkdown("##x").trim()).toBe("##x");
- expect(roundTripMarkdown("#a#b").trim()).toBe("#a#b");
+ // "#a#b" → text "#a" + tag "b"; the plain "#a" is tag-shaped, so the
+ // serializer escapes it to keep it literal. Doc-equivalent, stable after.
+ expect(roundTripMarkdown("#a#b").trim()).toBe("\\#a#b");
+ expect(roundTripMarkdown("\\#a#b").trim()).toBe("\\#a#b");
});
it("treats a run longer than 100 tag characters as plain text", () => {
@@ -109,9 +113,29 @@ describe("Tag mark", () => {
expect(textNodes.some((n) => hasTagMark(n))).toBe(false);
});
- it("documents known gap: backslash-escaped \\# degrades after one cycle (upstream escape dropping)", () => {
- const out = roundTripMarkdown("\\#escaped");
- expect(out.trim()).toBe("#escaped"); // upstream drops the escape (pre-existing); re-parse then sees a tag
- expect(roundTripMarkdown(out)).toBe(out); // at least stable from then on
+ it("keeps a backslash-escaped \\#tag literal and durable across round-trips", () => {
+ const input = "\\#NAS is my server";
+
+ // Escapes are lexical: `\#NAS` parses to ordinary text — no tag mark, no
+ // bespoke "escaped tag" node — the `\` is simply consumed.
+ const children = firstParagraphChildren(input);
+ expect(children.some(hasTagMark)).toBe(false);
+ expect(children.map((n) => n.text ?? "").join("")).toBe("#NAS is my server");
+
+ // The serializer re-escapes the tag-shaped `#`, so it never degrades into a
+ // tag...
+ const once = roundTripMarkdown(input).trim();
+ expect(once).toBe("\\#NAS is my server");
+ // ...and stays stable on every subsequent cycle.
+ expect(roundTripMarkdown(once).trim()).toBe(once);
+ });
+
+ it("escapes only the literal tag while still tagging a real one beside it", () => {
+ const input = "\\#NAS and #real";
+ expect(roundTripMarkdown(input).trim()).toBe("\\#NAS and #real");
+
+ const tagged = firstParagraphChildren(input).filter(hasTagMark);
+ expect(tagged).toHaveLength(1);
+ expect(tagged[0]).toMatchObject({ text: "#real", marks: [{ type: "tag", attrs: { tag: "real" } }] });
});
});
diff --git a/web/tests/remark-tag.test.tsx b/web/tests/remark-tag.test.tsx
index c4d26c90f..9b89e8f16 100644
--- a/web/tests/remark-tag.test.tsx
+++ b/web/tests/remark-tag.test.tsx
@@ -51,4 +51,31 @@ describe("remarkTag", () => {
expect(html).toContain('data-tag="urgent"');
expect(html).toContain('data-tag="later"');
});
+
+ it("does not turn a backslash-escaped \\#tag into a tag, but still tags an unescaped one", () => {
+ const html = renderMarkdown("\\#NAS is my server and a #real tag");
+
+ // Escaped: rendered as the literal text "#NAS", never a tag pill.
+ expect(html).not.toContain('data-tag="NAS"');
+ expect(html).toContain("#NAS");
+ // Unescaped neighbour is unaffected.
+ expect(html).toContain('data-tag="real"');
+ });
+
+ it("escapes only the marked hash when escaped and unescaped tags share a node", () => {
+ const html = renderMarkdown("\\#first then #second");
+
+ expect(html).not.toContain('data-tag="first"');
+ expect(html).toContain("#first");
+ expect(html).toContain('data-tag="second"');
+ });
+
+ it("still tags a hash that shares a text node with an entity reference", () => {
+ // The source slice ("...&amp;...") differs from the decoded value, so the
+ // escape-aware path bows out and the tag is detected the original way.
+ const html = renderMarkdown("Tom &amp; Jerry #cartoon");
+
+ expect(html).toContain('data-tag="cartoon"');
+ expect(html).toContain("Tom &amp; Jerry");
+ });
});