Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

fix(parser): support Unicode characters in tags

Steven64e9d82d675f
diff --git a/plugin/markdown/markdown_test.go b/plugin/markdown/markdown_test.go
index 628d96c4a..21a9f08cf 100644
--- a/plugin/markdown/markdown_test.go
+++ b/plugin/markdown/markdown_test.go
@@ -261,6 +261,42 @@ func TestExtractTags(t *testing.T) {
withExt: false,
expected: []string{},
},
+ {
+ name: "Chinese tag",
+ content: "Text with #测试",
+ withExt: true,
+ expected: []string{"测试"},
+ },
+ {
+ name: "Chinese tag followed by punctuation",
+ content: "Text #测试。 More text",
+ withExt: true,
+ expected: []string{"测试"},
+ },
+ {
+ name: "mixed Chinese and ASCII tag",
+ content: "#测试test123 content",
+ withExt: true,
+ expected: []string{"测试test123"},
+ },
+ {
+ name: "Japanese tag",
+ content: "#日本語 content",
+ withExt: true,
+ expected: []string{"日本語"},
+ },
+ {
+ name: "Korean tag",
+ content: "#한국어 content",
+ withExt: true,
+ expected: []string{"한국어"},
+ },
+ {
+ name: "hierarchical tag with Chinese",
+ content: "#work/测试/项目",
+ withExt: true,
+ expected: []string{"work/测试/项目"},
+ },
}
for _, tt := range tests {
diff --git a/plugin/markdown/parser/tag.go b/plugin/markdown/parser/tag.go
index 26804e266..038252159 100644
--- a/plugin/markdown/parser/tag.go
+++ b/plugin/markdown/parser/tag.go
@@ -45,20 +45,68 @@ func (*tagParser) Parse(_ gast.Node, block text.Reader, _ parser.Context) gast.N
}
// Scan tag characters
- // Valid: alphanumeric, dash, underscore, forward slash
+ // Tags include Unicode letters, digits, underscore, hyphen, forward slash
+ // Stop at: whitespace, punctuation (except - _ /)
+ // This follows the Twitter/social media standard for hashtag parsing
tagEnd := 1 // Start after #
for tagEnd < len(line) {
c := line[tagEnd]
- isValid := (c >= 'a' && c <= 'z') ||
- (c >= 'A' && c <= 'Z') ||
- (c >= '0' && c <= '9') ||
- c == '-' || c == '_' || c == '/'
+ // ASCII fast path for common characters
+ if (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') ||
+ (c >= '0' && c <= '9') || c == '_' || c == '-' || c == '/' {
+ tagEnd++
+ continue
+ }
+
+ // Stop at whitespace
+ if c == ' ' || c == '\t' || c == '\n' || c == '\r' {
+ break
+ }
- if !isValid {
+ // Stop at common ASCII punctuation
+ if c == '.' || c == ',' || c == ';' || c == ':' ||
+ c == '!' || c == '?' || c == '(' || c == ')' ||
+ c == '[' || c == ']' || c == '{' || c == '}' ||
+ c == '<' || c == '>' || c == '"' || c == '\'' ||
+ c == '`' || c == '|' || c == '\\' || c == '@' ||
+ c == '&' || c == '*' || c == '+' || c == '=' ||
+ c == '^' || c == '%' || c == '$' || c == '~' || c == '#' {
break
}
+ // For UTF-8 multibyte sequences, check for Unicode punctuation
+ // U+3000 (IDEOGRAPHIC SPACE) - treat as space
+ // U+3001-U+303F - CJK punctuation
+ // U+FF00-U+FFEF - Fullwidth punctuation
+ if c >= 0x80 && tagEnd+2 < len(line) {
+ b1, b2, b3 := line[tagEnd], line[tagEnd+1], line[tagEnd+2]
+
+ // U+3000 IDEOGRAPHIC SPACE (E3 80 80)
+ if b1 == 0xE3 && b2 == 0x80 && b3 == 0x80 {
+ break
+ }
+
+ // U+3001-U+303F CJK punctuation (E3 80 81 to E3 80 BF)
+ if b1 == 0xE3 && b2 == 0x80 && b3 >= 0x81 && b3 <= 0xBF {
+ break
+ }
+
+ // Common fullwidth punctuation: !?,。;:()
+ // U+FF01 ! (EF BC 81), U+FF1F ? (EF BC 9F)
+ // U+FF0C , (EF BC 8C), U+FF0E 。 (EF BC 8E)
+ // U+FF1A : (EF BC 9A), U+FF1B ; (EF BC 9B)
+ // U+FF08 ( (EF BC 88), U+FF09 ) (EF BC 89)
+ if b1 == 0xEF && b2 == 0xBC {
+ if b3 == 0x81 || b3 == 0x88 || b3 == 0x89 ||
+ b3 == 0x8C || b3 == 0x8E ||
+ b3 == 0x9A || b3 == 0x9B || b3 == 0x9F {
+ break
+ }
+ }
+ }
+
+ // Allow Unicode letters and other characters
tagEnd++
}
diff --git a/plugin/markdown/parser/tag_test.go b/plugin/markdown/parser/tag_test.go
index d238b591f..6150c4487 100644
--- a/plugin/markdown/parser/tag_test.go
+++ b/plugin/markdown/parser/tag_test.go
@@ -88,7 +88,7 @@ func TestTagParser(t *testing.T) {
name: "special characters",
input: "#tag@special",
expectedTag: "tag",
- shouldParse: true, // Stops at @
+ shouldParse: true,
},
{
name: "mixed case",
@@ -126,6 +126,48 @@ func TestTagParser(t *testing.T) {
expectedTag: "work-log/2024/q1",
shouldParse: true,
},
+ {
+ name: "Chinese characters",
+ input: "#测试",
+ expectedTag: "测试",
+ shouldParse: true,
+ },
+ {
+ name: "Chinese tag followed by space",
+ input: "#测试 some text",
+ expectedTag: "测试",
+ shouldParse: true,
+ },
+ {
+ name: "Chinese tag followed by punctuation",
+ input: "#测试。",
+ expectedTag: "测试",
+ shouldParse: true,
+ },
+ {
+ name: "mixed Chinese and ASCII",
+ input: "#测试test123",
+ expectedTag: "测试test123",
+ shouldParse: true,
+ },
+ {
+ name: "Japanese characters",
+ input: "#テスト",
+ expectedTag: "テスト",
+ shouldParse: true,
+ },
+ {
+ name: "Korean characters",
+ input: "#테스트",
+ expectedTag: "테스트",
+ shouldParse: true,
+ },
+ {
+ name: "emoji",
+ input: "#test🚀",
+ expectedTag: "test🚀",
+ shouldParse: true,
+ },
}
for _, tt := range tests {
diff --git a/web/src/utils/remark-plugins/remark-tag.ts b/web/src/utils/remark-plugins/remark-tag.ts
index ea3cd81fd..c2bfaf855 100644
--- a/web/src/utils/remark-plugins/remark-tag.ts
+++ b/web/src/utils/remark-plugins/remark-tag.ts
@@ -21,9 +21,28 @@ import { visit } from "unist-util-visit";
/**
* Check if character is valid for tag content
+ * Follows Twitter/social media standard: Unicode letters/digits, underscore, hyphen, slash
+ * Stops at whitespace and punctuation
*/
function isTagChar(char: string): boolean {
- return /[a-zA-Z0-9_\-/]/.test(char);
+ // Allow: letters (Unicode), digits (Unicode), underscore, hyphen, forward slash
+ // Stop at: whitespace, punctuation
+
+ // Stop at whitespace
+ if (/\s/.test(char)) {
+ return false;
+ }
+
+ // Stop at common punctuation (ASCII and Unicode)
+ // U+3000-U+303F: CJK punctuation
+ // U+FF00-U+FF65: Fullwidth punctuation subset
+ const punctuation = /[.,;:!?()[\]{}<>"'`|\\@&*+=^%$~#\u3000-\u303F\uFF00-\uFF0F\uFF1A-\uFF20\uFF3B-\uFF40\uFF5B-\uFF65]/;
+ if (punctuation.test(char)) {
+ return false;
+ }
+
+ // Allow everything else (Unicode letters, digits, and allowed symbols like - _ /)
+ return true;
}
/**