Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
feat: implement gomark parsers
Steven453707d18cdf
diff --git a/plugin/gomark/ast/block.go b/plugin/gomark/ast/block.goindex 42bf6f4ba..7eef58236 100644--- a/plugin/gomark/ast/block.go+++ b/plugin/gomark/ast/block.go@@ -1,6 +1,7 @@package asttype BaseBlock struct {+ Node}type LineBreak struct {@@ -50,3 +51,28 @@ var NodeTypeHeading = NewNodeType("Heading")func (*Heading) Type() NodeType {return NodeTypeHeading}++type HorizontalRule struct {+ BaseBlock++ // Symbol is "*" or "-" or "_".+ Symbol string+}++var NodeTypeHorizontalRule = NewNodeType("HorizontalRule")++func (*HorizontalRule) Type() NodeType {+ return NodeTypeHorizontalRule+}++type Blockquote struct {+ BaseBlock++ Children []Node+}++var NodeTypeBlockquote = NewNodeType("Blockquote")++func (*Blockquote) Type() NodeType {+ return NodeTypeBlockquote+}diff --git a/plugin/gomark/ast/inline.go b/plugin/gomark/ast/inline.goindex 3e4c57e57..d21fe1a16 100644--- a/plugin/gomark/ast/inline.go+++ b/plugin/gomark/ast/inline.go@@ -1,6 +1,8 @@package ast-type BaseInline struct{}+type BaseInline struct {+ Node+}type Text struct {BaseInline@@ -28,6 +30,34 @@ func (*Bold) Type() NodeType {return NodeTypeBold}+type Italic struct {+ BaseInline++ // Symbol is "*" or "_"+ Symbol string+ Content string+}++var NodeTypeItalic = NewNodeType("Italic")++func (*Italic) Type() NodeType {+ return NodeTypeItalic+}++type BoldItalic struct {+ BaseInline++ // Symbol is "*" or "_"+ Symbol string+ Content string+}++var NodeTypeBoldItalic = NewNodeType("BoldItalic")++func (*BoldItalic) Type() NodeType {+ return NodeTypeBoldItalic+}+type Code struct {BaseInline@@ -66,28 +96,26 @@ func (*Link) Type() NodeType {return NodeTypeLink}-type Italic struct {+type Tag struct {BaseInline- // Symbol is "*" or "_"- Symbol stringContent string}-var NodeTypeItalic = NewNodeType("Italic")+var NodeTypeTag = NewNodeType("Tag")-func (*Italic) Type() NodeType {- return NodeTypeItalic+func (*Tag) Type() NodeType {+ return NodeTypeTag}-type Tag struct {+type Strikethrough struct {BaseInlineContent string}-var NodeTypeTag = NewNodeType("Tag")+var NodeTypeStrikethrough = NewNodeType("Strikethrough")-func (*Tag) Type() NodeType {- return NodeTypeTag+func (*Strikethrough) Type() NodeType {+ return NodeTypeStrikethrough}diff --git a/plugin/gomark/parser/blockquote.go b/plugin/gomark/parser/blockquote.gonew file mode 100644index 000000000..fa9425c40--- /dev/null+++ b/plugin/gomark/parser/blockquote.go@@ -0,0 +1,47 @@+package parser++import (+ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++type BlockquoteParser struct{}++func NewBlockquoteParser() *BlockquoteParser {+ return &BlockquoteParser{}+}++func (*BlockquoteParser) Match(tokens []*tokenizer.Token) (int, bool) {+ if len(tokens) < 4 {+ return 0, false+ }+ if tokens[0].Type != tokenizer.GreaterThan || tokens[1].Type != tokenizer.Space {+ return 0, false+ }++ contentTokens := []*tokenizer.Token{}+ for _, token := range tokens[2:] {+ if token.Type == tokenizer.Newline {+ break+ }+ contentTokens = append(contentTokens, token)+ }+ if len(contentTokens) == 0 {+ return 0, false+ }++ return len(contentTokens) + 2, true+}++func (p *BlockquoteParser) Parse(tokens []*tokenizer.Token) ast.Node {+ size, ok := p.Match(tokens)+ if size == 0 || !ok {+ return nil+ }++ contentTokens := tokens[2:size]+ children := ParseInline(contentTokens)+ return &ast.Blockquote{+ Children: children,+ }+}diff --git a/plugin/gomark/parser/blockquote_test.go b/plugin/gomark/parser/blockquote_test.gonew file mode 100644index 000000000..2e0f2ae0a--- /dev/null+++ b/plugin/gomark/parser/blockquote_test.go@@ -0,0 +1,47 @@+package parser++import (+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++func TestBlockquoteParser(t *testing.T) {+ tests := []struct {+ text string+ blockquote ast.Node+ }{+ {+ text: "> Hello world",+ blockquote: &ast.Blockquote{+ Children: []ast.Node{+ &ast.Text{+ Content: "Hello world",+ },+ },+ },+ },+ {+ text: "> Hello\nworld",+ blockquote: &ast.Blockquote{+ Children: []ast.Node{+ &ast.Text{+ Content: "Hello",+ },+ },+ },+ },+ {+ text: ">Hello\nworld",+ blockquote: nil,+ },+ }++ for _, test := range tests {+ tokens := tokenizer.Tokenize(test.text)+ require.Equal(t, test.blockquote, NewBlockquoteParser().Parse(tokens))+ }+}diff --git a/plugin/gomark/parser/bold.go b/plugin/gomark/parser/bold.goindex 237cb8857..36cf24340 100644--- a/plugin/gomark/parser/bold.go+++ b/plugin/gomark/parser/bold.go@@ -7,10 +7,8 @@ import (type BoldParser struct{}-var defaultBoldParser = &BoldParser{}-func NewBoldParser() InlineParser {- return defaultBoldParser+ return &BoldParser{}}func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) {@@ -23,7 +21,7 @@ func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) {return 0, false}prefixTokenType := prefixTokens[0].Type- if prefixTokenType != tokenizer.Star && prefixTokenType != tokenizer.Underline {+ if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {return 0, false}diff --git a/plugin/gomark/parser/bold_italic.go b/plugin/gomark/parser/bold_italic.gonew file mode 100644index 000000000..a97365942--- /dev/null+++ b/plugin/gomark/parser/bold_italic.go@@ -0,0 +1,58 @@+package parser++import (+ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++type BoldItalicParser struct{}++func NewBoldItalicParser() InlineParser {+ return &BoldItalicParser{}+}++func (*BoldItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {+ if len(tokens) < 7 {+ return 0, false+ }++ prefixTokens := tokens[:3]+ if prefixTokens[0].Type != prefixTokens[1].Type || prefixTokens[0].Type != prefixTokens[2].Type || prefixTokens[1].Type != prefixTokens[2].Type {+ return 0, false+ }+ prefixTokenType := prefixTokens[0].Type+ if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {+ return 0, false+ }++ cursor, matched := 3, false+ for ; cursor < len(tokens)-2; cursor++ {+ token, nextToken, endToken := tokens[cursor], tokens[cursor+1], tokens[cursor+2]+ if token.Type == tokenizer.Newline || nextToken.Type == tokenizer.Newline || endToken.Type == tokenizer.Newline {+ return 0, false+ }+ if token.Type == prefixTokenType && nextToken.Type == prefixTokenType && endToken.Type == prefixTokenType {+ matched = true+ break+ }+ }+ if !matched {+ return 0, false+ }++ return cursor + 3, true+}++func (p *BoldItalicParser) Parse(tokens []*tokenizer.Token) ast.Node {+ size, ok := p.Match(tokens)+ if size == 0 || !ok {+ return nil+ }++ prefixTokenType := tokens[0].Type+ contentTokens := tokens[3 : size-3]+ return &ast.BoldItalic{+ Symbol: prefixTokenType,+ Content: tokenizer.Stringify(contentTokens),+ }+}diff --git a/plugin/gomark/parser/bold_italic_test.go b/plugin/gomark/parser/bold_italic_test.gonew file mode 100644index 000000000..855798593--- /dev/null+++ b/plugin/gomark/parser/bold_italic_test.go@@ -0,0 +1,49 @@+package parser++import (+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++func TestBoldItalicParser(t *testing.T) {+ tests := []struct {+ text string+ boldItalic ast.Node+ }{+ {+ text: "*Hello world!",+ boldItalic: nil,+ },+ {+ text: "***Hello***",+ boldItalic: &ast.BoldItalic{+ Symbol: "*",+ Content: "Hello",+ },+ },+ {+ text: "*** Hello ***",+ boldItalic: &ast.BoldItalic{+ Symbol: "*",+ Content: " Hello ",+ },+ },+ {+ text: "*** Hello * *",+ boldItalic: nil,+ },+ {+ text: "*** Hello **",+ boldItalic: nil,+ },+ }++ for _, test := range tests {+ tokens := tokenizer.Tokenize(test.text)+ require.Equal(t, test.boldItalic, NewBoldItalicParser().Parse(tokens))+ }+}diff --git a/plugin/gomark/parser/code.go b/plugin/gomark/parser/code.goindex 903b1cf77..b6dff80c6 100644--- a/plugin/gomark/parser/code.go+++ b/plugin/gomark/parser/code.go@@ -7,10 +7,8 @@ import (type CodeParser struct{}-var defaultCodeParser = &CodeParser{}-func NewCodeParser() *CodeParser {- return defaultCodeParser+ return &CodeParser{}}func (*CodeParser) Match(tokens []*tokenizer.Token) (int, bool) {diff --git a/plugin/gomark/parser/code_block.go b/plugin/gomark/parser/code_block.goindex 7cc8510fd..8ae48176f 100644--- a/plugin/gomark/parser/code_block.go+++ b/plugin/gomark/parser/code_block.go@@ -10,10 +10,8 @@ type CodeBlockParser struct {Content string}-var defaultCodeBlockParser = &CodeBlockParser{}-func NewCodeBlockParser() *CodeBlockParser {- return defaultCodeBlockParser+ return &CodeBlockParser{}}func (*CodeBlockParser) Match(tokens []*tokenizer.Token) (int, bool) {diff --git a/plugin/gomark/parser/heading.go b/plugin/gomark/parser/heading.goindex 8becfc5f7..eaefcb333 100644--- a/plugin/gomark/parser/heading.go+++ b/plugin/gomark/parser/heading.go@@ -62,11 +62,7 @@ func (p *HeadingParser) Parse(tokens []*tokenizer.Token) ast.Node {}}contentTokens := tokens[level+1 : size]- children := ParseInline(contentTokens, []InlineParser{- NewBoldParser(),- NewCodeParser(),- NewTextParser(),- })+ children := ParseInline(contentTokens)return &ast.Heading{Level: level,Children: children,diff --git a/plugin/gomark/parser/horizontal_rule.go b/plugin/gomark/parser/horizontal_rule.gonew file mode 100644index 000000000..a008d4433--- /dev/null+++ b/plugin/gomark/parser/horizontal_rule.go@@ -0,0 +1,39 @@+package parser++import (+ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++type HorizontalRuleParser struct{}++func NewHorizontalRuleParser() *HorizontalRuleParser {+ return &HorizontalRuleParser{}+}++func (*HorizontalRuleParser) Match(tokens []*tokenizer.Token) (int, bool) {+ if len(tokens) < 3 {+ return 0, false+ }+ if tokens[0].Type != tokens[1].Type || tokens[0].Type != tokens[2].Type || tokens[1].Type != tokens[2].Type {+ return 0, false+ }+ if tokens[0].Type != tokenizer.Dash && tokens[0].Type != tokenizer.Underline && tokens[0].Type != tokenizer.Asterisk {+ return 0, false+ }+ if len(tokens) > 3 && tokens[3].Type != tokenizer.Newline {+ return 0, false+ }+ return 3, true+}++func (p *HorizontalRuleParser) Parse(tokens []*tokenizer.Token) ast.Node {+ size, ok := p.Match(tokens)+ if size == 0 || !ok {+ return nil+ }++ return &ast.HorizontalRule{+ Symbol: tokens[0].Type,+ }+}diff --git a/plugin/gomark/parser/horizontal_rule_test.go b/plugin/gomark/parser/horizontal_rule_test.gonew file mode 100644index 000000000..f45878f1a--- /dev/null+++ b/plugin/gomark/parser/horizontal_rule_test.go@@ -0,0 +1,49 @@+package parser++import (+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++func TestHorizontalRuleParser(t *testing.T) {+ tests := []struct {+ text string+ horizontalRule ast.Node+ }{+ {+ text: "---",+ horizontalRule: &ast.HorizontalRule{+ Symbol: "-",+ },+ },+ {+ text: "****",+ horizontalRule: nil,+ },+ {+ text: "***",+ horizontalRule: &ast.HorizontalRule{+ Symbol: "*",+ },+ },+ {+ text: "-*-",+ horizontalRule: nil,+ },+ {+ text: "___",+ horizontalRule: &ast.HorizontalRule{+ Symbol: "_",+ },+ },+ }++ for _, test := range tests {+ tokens := tokenizer.Tokenize(test.text)+ require.Equal(t, test.horizontalRule, NewHorizontalRuleParser().Parse(tokens))+ }+}diff --git a/plugin/gomark/parser/image.go b/plugin/gomark/parser/image.goindex 713981203..83be30bdf 100644--- a/plugin/gomark/parser/image.go+++ b/plugin/gomark/parser/image.go@@ -7,10 +7,8 @@ import (type ImageParser struct{}-var defaultImageParser = &ImageParser{}-func NewImageParser() *ImageParser {- return defaultImageParser+ return &ImageParser{}}func (*ImageParser) Match(tokens []*tokenizer.Token) (int, bool) {diff --git a/plugin/gomark/parser/italic.go b/plugin/gomark/parser/italic.goindex 3ab01f935..e9e32fefd 100644--- a/plugin/gomark/parser/italic.go+++ b/plugin/gomark/parser/italic.go@@ -1,6 +1,9 @@package parser-import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+import (+ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)type ItalicParser struct {ContentTokens []*tokenizer.Token@@ -10,21 +13,21 @@ func NewItalicParser() *ItalicParser {return &ItalicParser{}}-func (*ItalicParser) Match(tokens []*tokenizer.Token) *ItalicParser {+func (*ItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {if len(tokens) < 3 {- return nil+ return 0, false}prefixTokens := tokens[:1]- if prefixTokens[0].Type != tokenizer.Star && prefixTokens[0].Type != tokenizer.Underline {- return nil+ if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underline {+ return 0, false}prefixTokenType := prefixTokens[0].TypecontentTokens := []*tokenizer.Token{}matched := falsefor _, token := range tokens[1:] {if token.Type == tokenizer.Newline {- return nil+ return 0, false}if token.Type == prefixTokenType {matched = true@@ -33,10 +36,22 @@ func (*ItalicParser) Match(tokens []*tokenizer.Token) *ItalicParser {contentTokens = append(contentTokens, token)}if !matched || len(contentTokens) == 0 {+ return 0, false+ }++ return len(contentTokens) + 2, true+}++func (p *ItalicParser) Parse(tokens []*tokenizer.Token) ast.Node {+ size, ok := p.Match(tokens)+ if size == 0 || !ok {return nil}- return &ItalicParser{- ContentTokens: contentTokens,+ prefixTokenType := tokens[0].Type+ contentTokens := tokens[1 : size-1]+ return &ast.Italic{+ Symbol: prefixTokenType,+ Content: tokenizer.Stringify(contentTokens),}}diff --git a/plugin/gomark/parser/italic_test.go b/plugin/gomark/parser/italic_test.goindex 768931231..60c6ae534 100644--- a/plugin/gomark/parser/italic_test.go+++ b/plugin/gomark/parser/italic_test.go@@ -5,13 +5,14 @@ import ("github.com/stretchr/testify/require"+ "github.com/usememos/memos/plugin/gomark/ast""github.com/usememos/memos/plugin/gomark/parser/tokenizer")func TestItalicParser(t *testing.T) {tests := []struct {text string- italic *ItalicParser+ italic ast.Node}{{text: "*Hello world!",@@ -19,76 +20,29 @@ func TestItalicParser(t *testing.T) {},{text: "*Hello*",- italic: &ItalicParser{- ContentTokens: []*tokenizer.Token{- {- Type: tokenizer.Text,- Value: "Hello",- },- },+ italic: &ast.Italic{+ Symbol: "*",+ Content: "Hello",},},{text: "* Hello *",- italic: &ItalicParser{- ContentTokens: []*tokenizer.Token{- {- Type: tokenizer.Space,- Value: " ",- },- {- Type: tokenizer.Text,- Value: "Hello",- },- {- Type: tokenizer.Space,- Value: " ",- },- },+ italic: &ast.Italic{+ Symbol: "*",+ Content: " Hello ",},},- {- text: "** Hello * *",- italic: nil,- },{text: "*1* Hello * *",- italic: &ItalicParser{- ContentTokens: []*tokenizer.Token{- {- Type: tokenizer.Text,- Value: "1",- },- },+ italic: &ast.Italic{+ Symbol: "*",+ Content: "1",},},- {- text: `* \n * Hello * *`,- italic: &ItalicParser{- ContentTokens: []*tokenizer.Token{- {- Type: tokenizer.Space,- Value: " ",- },- {- Type: tokenizer.Text,- Value: `\n`,- },- {- Type: tokenizer.Space,- Value: " ",- },- },- },- },- {- text: "* \n * Hello * *",- italic: nil,- },}for _, test := range tests {tokens := tokenizer.Tokenize(test.text)- require.Equal(t, test.italic, NewItalicParser().Match(tokens))+ require.Equal(t, test.italic, NewItalicParser().Parse(tokens))}}diff --git a/plugin/gomark/parser/line_break.go b/plugin/gomark/parser/line_break.goindex bdc0957d6..6017f72a7 100644--- a/plugin/gomark/parser/line_break.go+++ b/plugin/gomark/parser/line_break.go@@ -7,10 +7,8 @@ import (type LineBreakParser struct{}-var defaultLineBreakParser = &LineBreakParser{}-func NewLineBreakParser() *LineBreakParser {- return defaultLineBreakParser+ return &LineBreakParser{}}func (*LineBreakParser) Match(tokens []*tokenizer.Token) (int, bool) {diff --git a/plugin/gomark/parser/link.go b/plugin/gomark/parser/link.goindex a1c8b7449..3b5fd6439 100644--- a/plugin/gomark/parser/link.go+++ b/plugin/gomark/parser/link.go@@ -1,58 +1,72 @@package parser-import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+import (+ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)-type LinkParser struct {- ContentTokens []*tokenizer.Token- URL string-}+type LinkParser struct{}func NewLinkParser() *LinkParser {return &LinkParser{}}-func (*LinkParser) Match(tokens []*tokenizer.Token) *LinkParser {- if len(tokens) < 4 {- return nil+func (*LinkParser) Match(tokens []*tokenizer.Token) (int, bool) {+ if len(tokens) < 5 {+ return 0, false}if tokens[0].Type != tokenizer.LeftSquareBracket {- return nil+ return 0, false}- cursor, contentTokens := 1, []*tokenizer.Token{}- for ; cursor < len(tokens)-2; cursor++ {- if tokens[cursor].Type == tokenizer.Newline {- return nil+ textTokens := []*tokenizer.Token{}+ for _, token := range tokens[1:] {+ if token.Type == tokenizer.Newline {+ return 0, false}- if tokens[cursor].Type == tokenizer.RightSquareBracket {+ if token.Type == tokenizer.RightSquareBracket {break}- contentTokens = append(contentTokens, tokens[cursor])+ textTokens = append(textTokens, token)}- if tokens[cursor+1].Type != tokenizer.LeftParenthesis {- return nil+ if len(textTokens)+4 >= len(tokens) {+ return 0, false+ }+ if tokens[2+len(textTokens)].Type != tokenizer.LeftParenthesis {+ return 0, false}- matched, url := false, ""- for _, token := range tokens[cursor+2:] {+ urlTokens := []*tokenizer.Token{}+ for _, token := range tokens[3+len(textTokens):] {if token.Type == tokenizer.Newline || token.Type == tokenizer.Space {- return nil+ return 0, false}if token.Type == tokenizer.RightParenthesis {- matched = truebreak}- url += token.Value+ urlTokens = append(urlTokens, token)+ }+ if 4+len(urlTokens)+len(textTokens) > len(tokens) {+ return 0, false}- if !matched || url == "" {++ return 4 + len(urlTokens) + len(textTokens), true+}++func (p *LinkParser) Parse(tokens []*tokenizer.Token) ast.Node {+ size, ok := p.Match(tokens)+ if size == 0 || !ok {return nil}- if len(contentTokens) == 0 {- contentTokens = append(contentTokens, &tokenizer.Token{- Type: tokenizer.Text,- Value: url,- })++ textTokens := []*tokenizer.Token{}+ for _, token := range tokens[1:] {+ if token.Type == tokenizer.RightSquareBracket {+ break+ }+ textTokens = append(textTokens, token)}- return &LinkParser{- ContentTokens: contentTokens,- URL: url,+ urlTokens := tokens[2+len(textTokens)+1 : size-1]+ return &ast.Link{+ Text: tokenizer.Stringify(textTokens),+ URL: tokenizer.Stringify(urlTokens),}}diff --git a/plugin/gomark/parser/link_test.go b/plugin/gomark/parser/link_test.goindex ab4f5c7dc..421ab38a5 100644--- a/plugin/gomark/parser/link_test.go+++ b/plugin/gomark/parser/link_test.go@@ -5,24 +5,20 @@ import ("github.com/stretchr/testify/require"+ "github.com/usememos/memos/plugin/gomark/ast""github.com/usememos/memos/plugin/gomark/parser/tokenizer")func TestLinkParser(t *testing.T) {tests := []struct {text string- link *LinkParser+ link ast.Node}{{text: "[](https://example.com)",- link: &LinkParser{- ContentTokens: []*tokenizer.Token{- {- Type: tokenizer.Text,- Value: "https://example.com",- },- },- URL: "https://example.com",+ link: &ast.Link{+ Text: "",+ URL: "https://example.com",},},{@@ -35,27 +31,14 @@ func TestLinkParser(t *testing.T) {},{text: "[hello world](https://example.com)",- link: &LinkParser{- ContentTokens: []*tokenizer.Token{- {- Type: tokenizer.Text,- Value: "hello",- },- {- Type: tokenizer.Space,- Value: " ",- },- {- Type: tokenizer.Text,- Value: "world",- },- },- URL: "https://example.com",+ link: &ast.Link{+ Text: "hello world",+ URL: "https://example.com",},},}for _, test := range tests {tokens := tokenizer.Tokenize(test.text)- require.Equal(t, test.link, NewLinkParser().Match(tokens))+ require.Equal(t, test.link, NewLinkParser().Parse(tokens))}}diff --git a/plugin/gomark/parser/paragraph.go b/plugin/gomark/parser/paragraph.goindex c2403ad02..09e69047f 100644--- a/plugin/gomark/parser/paragraph.go+++ b/plugin/gomark/parser/paragraph.go@@ -9,10 +9,8 @@ type ParagraphParser struct {ContentTokens []*tokenizer.Token}-var defaultParagraphParser = &ParagraphParser{}-func NewParagraphParser() *ParagraphParser {- return defaultParagraphParser+ return &ParagraphParser{}}func (*ParagraphParser) Match(tokens []*tokenizer.Token) (int, bool) {@@ -38,10 +36,7 @@ func (p *ParagraphParser) Parse(tokens []*tokenizer.Token) ast.Node {}contentTokens := tokens[:size]- children := ParseInline(contentTokens, []InlineParser{- NewBoldParser(),- NewTextParser(),- })+ children := ParseInline(contentTokens)return &ast.Paragraph{Children: children,}diff --git a/plugin/gomark/parser/parser.go b/plugin/gomark/parser/parser.goindex c82fa50d5..123eead34 100644--- a/plugin/gomark/parser/parser.go+++ b/plugin/gomark/parser/parser.go@@ -1,6 +1,8 @@package parserimport (+ "errors"+"github.com/usememos/memos/plugin/gomark/ast""github.com/usememos/memos/plugin/gomark/parser/tokenizer")@@ -23,32 +25,51 @@ type BlockParser interface {BaseParser}-func Parse(tokens []*tokenizer.Token) []ast.Node {+var defaultBlockParsers = []BlockParser{+ NewCodeBlockParser(),+ NewHorizontalRuleParser(),+ NewHeadingParser(),+ NewBlockquoteParser(),+ NewParagraphParser(),+ NewLineBreakParser(),+}++func Parse(tokens []*tokenizer.Token) ([]ast.Node, error) {nodes := []ast.Node{}- blockParsers := []BlockParser{- NewCodeBlockParser(),- NewParagraphParser(),- NewLineBreakParser(),- }for len(tokens) > 0 {- for _, blockParser := range blockParsers {+ for _, blockParser := range defaultBlockParsers {cursor, matched := blockParser.Match(tokens)if matched {node := blockParser.Parse(tokens)+ if node == nil {+ return nil, errors.New("parse error")+ }nodes = append(nodes, node)tokens = tokens[cursor:]break}}}- return nodes+ return nodes, nil+}++var defaultInlineParsers = []InlineParser{+ NewBoldItalicParser(),+ NewImageParser(),+ NewLinkParser(),+ NewBoldParser(),+ NewItalicParser(),+ NewCodeParser(),+ NewTagParser(),+ NewStrikethroughParser(),+ NewTextParser(),}-func ParseInline(tokens []*tokenizer.Token, inlineParsers []InlineParser) []ast.Node {+func ParseInline(tokens []*tokenizer.Token) []ast.Node {nodes := []ast.Node{}var lastNode ast.Nodefor len(tokens) > 0 {- for _, inlineParser := range inlineParsers {+ for _, inlineParser := range defaultInlineParsers {cursor, matched := inlineParser.Match(tokens)if matched {node := inlineParser.Parse(tokens)diff --git a/plugin/gomark/parser/parser_test.go b/plugin/gomark/parser/parser_test.goindex dc2d6e68a..62b06ab90 100644--- a/plugin/gomark/parser/parser_test.go+++ b/plugin/gomark/parser/parser_test.go@@ -89,6 +89,8 @@ func TestParser(t *testing.T) {for _, test := range tests {tokens := tokenizer.Tokenize(test.text)- require.Equal(t, test.nodes, Parse(tokens))+ nodes, err := Parse(tokens)+ require.NoError(t, err)+ require.Equal(t, test.nodes, nodes)}}diff --git a/plugin/gomark/parser/strikethrough.go b/plugin/gomark/parser/strikethrough.gonew file mode 100644index 000000000..fde8acc59--- /dev/null+++ b/plugin/gomark/parser/strikethrough.go@@ -0,0 +1,49 @@+package parser++import (+ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++type StrikethroughParser struct{}++func NewStrikethroughParser() *StrikethroughParser {+ return &StrikethroughParser{}+}++func (*StrikethroughParser) Match(tokens []*tokenizer.Token) (int, bool) {+ if len(tokens) < 5 {+ return 0, false+ }+ if tokens[0].Type != tokenizer.Tilde || tokens[1].Type != tokenizer.Tilde {+ return 0, false+ }++ cursor, matched := 2, false+ for ; cursor < len(tokens)-1; cursor++ {+ token, nextToken := tokens[cursor], tokens[cursor+1]+ if token.Type == tokenizer.Newline || nextToken.Type == tokenizer.Newline {+ return 0, false+ }+ if token.Type == tokenizer.Tilde && nextToken.Type == tokenizer.Tilde {+ matched = true+ break+ }+ }+ if !matched {+ return 0, false+ }+ return cursor + 2, true+}++func (p *StrikethroughParser) Parse(tokens []*tokenizer.Token) ast.Node {+ size, ok := p.Match(tokens)+ if size == 0 || !ok {+ return nil+ }++ contentTokens := tokens[2 : size-2]+ return &ast.Strikethrough{+ Content: tokenizer.Stringify(contentTokens),+ }+}diff --git a/plugin/gomark/parser/strikethrough_test.go b/plugin/gomark/parser/strikethrough_test.gonew file mode 100644index 000000000..28a04bcc6--- /dev/null+++ b/plugin/gomark/parser/strikethrough_test.go@@ -0,0 +1,45 @@+package parser++import (+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++func TestStrikethroughParser(t *testing.T) {+ tests := []struct {+ text string+ strikethrough ast.Node+ }{+ {+ text: "~~Hello world",+ strikethrough: nil,+ },+ {+ text: "~~Hello~~",+ strikethrough: &ast.Strikethrough{+ Content: "Hello",+ },+ },+ {+ text: "~~ Hello ~~",+ strikethrough: &ast.Strikethrough{+ Content: " Hello ",+ },+ },+ {+ text: "~~1~~ Hello ~~~",+ strikethrough: &ast.Strikethrough{+ Content: "1",+ },+ },+ }++ for _, test := range tests {+ tokens := tokenizer.Tokenize(test.text)+ require.Equal(t, test.strikethrough, NewStrikethroughParser().Parse(tokens))+ }+}diff --git a/plugin/gomark/parser/tag.go b/plugin/gomark/parser/tag.goindex c33fcd0b4..80fc15ed4 100644--- a/plugin/gomark/parser/tag.go+++ b/plugin/gomark/parser/tag.go@@ -1,21 +1,22 @@package parser-import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+import (+ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)-type TagParser struct {- ContentTokens []*tokenizer.Token-}+type TagParser struct{}func NewTagParser() *TagParser {return &TagParser{}}-func (*TagParser) Match(tokens []*tokenizer.Token) *TagParser {+func (*TagParser) Match(tokens []*tokenizer.Token) (int, bool) {if len(tokens) < 2 {- return nil+ return 0, false}if tokens[0].Type != tokenizer.Hash {- return nil+ return 0, false}contentTokens := []*tokenizer.Token{}for _, token := range tokens[1:] {@@ -25,10 +26,20 @@ func (*TagParser) Match(tokens []*tokenizer.Token) *TagParser {contentTokens = append(contentTokens, token)}if len(contentTokens) == 0 {+ return 0, false+ }++ return len(contentTokens) + 1, true+}++func (p *TagParser) Parse(tokens []*tokenizer.Token) ast.Node {+ size, ok := p.Match(tokens)+ if size == 0 || !ok {return nil}- return &TagParser{- ContentTokens: contentTokens,+ contentTokens := tokens[1:size]+ return &ast.Tag{+ Content: tokenizer.Stringify(contentTokens),}}diff --git a/plugin/gomark/parser/tag_test.go b/plugin/gomark/parser/tag_test.goindex 7066027f5..54d541e71 100644--- a/plugin/gomark/parser/tag_test.go+++ b/plugin/gomark/parser/tag_test.go@@ -5,13 +5,14 @@ import ("github.com/stretchr/testify/require"+ "github.com/usememos/memos/plugin/gomark/ast""github.com/usememos/memos/plugin/gomark/parser/tokenizer")func TestTagParser(t *testing.T) {tests := []struct {text string- tag *TagParser+ tag ast.Node}{{text: "*Hello world",@@ -23,30 +24,20 @@ func TestTagParser(t *testing.T) {},{text: "#tag",- tag: &TagParser{- ContentTokens: []*tokenizer.Token{- {- Type: tokenizer.Text,- Value: "tag",- },- },+ tag: &ast.Tag{+ Content: "tag",},},{- text: "#tag/subtag",- tag: &TagParser{- ContentTokens: []*tokenizer.Token{- {- Type: tokenizer.Text,- Value: "tag/subtag",- },- },+ text: "#tag/subtag 123",+ tag: &ast.Tag{+ Content: "tag/subtag",},},}for _, test := range tests {tokens := tokenizer.Tokenize(test.text)- require.Equal(t, test.tag, NewTagParser().Match(tokens))+ require.Equal(t, test.tag, NewTagParser().Parse(tokens))}}diff --git a/plugin/gomark/parser/text.go b/plugin/gomark/parser/text.goindex f930ad3dd..a2523ff29 100644--- a/plugin/gomark/parser/text.go+++ b/plugin/gomark/parser/text.go@@ -9,10 +9,8 @@ type TextParser struct {Content string}-var defaultTextParser = &TextParser{}-func NewTextParser() *TextParser {- return defaultTextParser+ return &TextParser{}}func (*TextParser) Match(tokens []*tokenizer.Token) (int, bool) {diff --git a/plugin/gomark/parser/tokenizer/tokenizer.go b/plugin/gomark/parser/tokenizer/tokenizer.goindex a9c9f42ce..f27ecf03b 100644--- a/plugin/gomark/parser/tokenizer/tokenizer.go+++ b/plugin/gomark/parser/tokenizer/tokenizer.go@@ -4,7 +4,7 @@ type TokenType = stringconst (Underline TokenType = "_"- Star TokenType = "*"+ Asterisk TokenType = "*"Hash TokenType = "#"Backtick TokenType = "`"LeftSquareBracket TokenType = "["@@ -12,6 +12,9 @@ const (LeftParenthesis TokenType = "("RightParenthesis TokenType = ")"ExclamationMark TokenType = "!"+ Tilde TokenType = "~"+ Dash TokenType = "-"+ GreaterThan TokenType = ">"Newline TokenType = "\n"Space TokenType = " ")@@ -39,7 +42,7 @@ func Tokenize(text string) []*Token {case '_':tokens = append(tokens, NewToken(Underline, "_"))case '*':- tokens = append(tokens, NewToken(Star, "*"))+ tokens = append(tokens, NewToken(Asterisk, "*"))case '#':tokens = append(tokens, NewToken(Hash, "#"))case '`':@@ -54,6 +57,12 @@ func Tokenize(text string) []*Token {tokens = append(tokens, NewToken(RightParenthesis, ")"))case '!':tokens = append(tokens, NewToken(ExclamationMark, "!"))+ case '~':+ tokens = append(tokens, NewToken(Tilde, "~"))+ case '-':+ tokens = append(tokens, NewToken(Dash, "-"))+ case '>':+ tokens = append(tokens, NewToken(GreaterThan, ">"))case '\n':tokens = append(tokens, NewToken(Newline, "\n"))case ' ':diff --git a/plugin/gomark/parser/tokenizer/tokenizer_test.go b/plugin/gomark/parser/tokenizer/tokenizer_test.goindex 8010fe4f6..a85651b3d 100644--- a/plugin/gomark/parser/tokenizer/tokenizer_test.go+++ b/plugin/gomark/parser/tokenizer/tokenizer_test.go@@ -15,7 +15,7 @@ func TestTokenize(t *testing.T) {text: "*Hello world!",tokens: []*Token{{- Type: Star,+ Type: Asterisk,Value: "*",},{