Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

feat: implement gomark parsers

Steven453707d18cdf
diff --git a/plugin/gomark/ast/block.go b/plugin/gomark/ast/block.go
index 42bf6f4ba..7eef58236 100644
--- a/plugin/gomark/ast/block.go
+++ b/plugin/gomark/ast/block.go
@@ -1,6 +1,7 @@
package ast
type BaseBlock struct {
+ Node
}
type LineBreak struct {
@@ -50,3 +51,28 @@ var NodeTypeHeading = NewNodeType("Heading")
func (*Heading) Type() NodeType {
return NodeTypeHeading
}
+
+type HorizontalRule struct {
+ BaseBlock
+
+ // Symbol is "*" or "-" or "_".
+ Symbol string
+}
+
+var NodeTypeHorizontalRule = NewNodeType("HorizontalRule")
+
+func (*HorizontalRule) Type() NodeType {
+ return NodeTypeHorizontalRule
+}
+
+type Blockquote struct {
+ BaseBlock
+
+ Children []Node
+}
+
+var NodeTypeBlockquote = NewNodeType("Blockquote")
+
+func (*Blockquote) Type() NodeType {
+ return NodeTypeBlockquote
+}
diff --git a/plugin/gomark/ast/inline.go b/plugin/gomark/ast/inline.go
index 3e4c57e57..d21fe1a16 100644
--- a/plugin/gomark/ast/inline.go
+++ b/plugin/gomark/ast/inline.go
@@ -1,6 +1,8 @@
package ast
-type BaseInline struct{}
+type BaseInline struct {
+ Node
+}
type Text struct {
BaseInline
@@ -28,6 +30,34 @@ func (*Bold) Type() NodeType {
return NodeTypeBold
}
+type Italic struct {
+ BaseInline
+
+ // Symbol is "*" or "_"
+ Symbol string
+ Content string
+}
+
+var NodeTypeItalic = NewNodeType("Italic")
+
+func (*Italic) Type() NodeType {
+ return NodeTypeItalic
+}
+
+type BoldItalic struct {
+ BaseInline
+
+ // Symbol is "*" or "_"
+ Symbol string
+ Content string
+}
+
+var NodeTypeBoldItalic = NewNodeType("BoldItalic")
+
+func (*BoldItalic) Type() NodeType {
+ return NodeTypeBoldItalic
+}
+
type Code struct {
BaseInline
@@ -66,28 +96,26 @@ func (*Link) Type() NodeType {
return NodeTypeLink
}
-type Italic struct {
+type Tag struct {
BaseInline
- // Symbol is "*" or "_"
- Symbol string
Content string
}
-var NodeTypeItalic = NewNodeType("Italic")
+var NodeTypeTag = NewNodeType("Tag")
-func (*Italic) Type() NodeType {
- return NodeTypeItalic
+func (*Tag) Type() NodeType {
+ return NodeTypeTag
}
-type Tag struct {
+type Strikethrough struct {
BaseInline
Content string
}
-var NodeTypeTag = NewNodeType("Tag")
+var NodeTypeStrikethrough = NewNodeType("Strikethrough")
-func (*Tag) Type() NodeType {
- return NodeTypeTag
+func (*Strikethrough) Type() NodeType {
+ return NodeTypeStrikethrough
}
diff --git a/plugin/gomark/parser/blockquote.go b/plugin/gomark/parser/blockquote.go
new file mode 100644
index 000000000..fa9425c40
--- /dev/null
+++ b/plugin/gomark/parser/blockquote.go
@@ -0,0 +1,47 @@
+package parser
+
+import (
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+type BlockquoteParser struct{}
+
+func NewBlockquoteParser() *BlockquoteParser {
+ return &BlockquoteParser{}
+}
+
+func (*BlockquoteParser) Match(tokens []*tokenizer.Token) (int, bool) {
+ if len(tokens) < 4 {
+ return 0, false
+ }
+ if tokens[0].Type != tokenizer.GreaterThan || tokens[1].Type != tokenizer.Space {
+ return 0, false
+ }
+
+ contentTokens := []*tokenizer.Token{}
+ for _, token := range tokens[2:] {
+ if token.Type == tokenizer.Newline {
+ break
+ }
+ contentTokens = append(contentTokens, token)
+ }
+ if len(contentTokens) == 0 {
+ return 0, false
+ }
+
+ return len(contentTokens) + 2, true
+}
+
+func (p *BlockquoteParser) Parse(tokens []*tokenizer.Token) ast.Node {
+ size, ok := p.Match(tokens)
+ if size == 0 || !ok {
+ return nil
+ }
+
+ contentTokens := tokens[2:size]
+ children := ParseInline(contentTokens)
+ return &ast.Blockquote{
+ Children: children,
+ }
+}
diff --git a/plugin/gomark/parser/blockquote_test.go b/plugin/gomark/parser/blockquote_test.go
new file mode 100644
index 000000000..2e0f2ae0a
--- /dev/null
+++ b/plugin/gomark/parser/blockquote_test.go
@@ -0,0 +1,47 @@
+package parser
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+func TestBlockquoteParser(t *testing.T) {
+ tests := []struct {
+ text string
+ blockquote ast.Node
+ }{
+ {
+ text: "> Hello world",
+ blockquote: &ast.Blockquote{
+ Children: []ast.Node{
+ &ast.Text{
+ Content: "Hello world",
+ },
+ },
+ },
+ },
+ {
+ text: "> Hello\nworld",
+ blockquote: &ast.Blockquote{
+ Children: []ast.Node{
+ &ast.Text{
+ Content: "Hello",
+ },
+ },
+ },
+ },
+ {
+ text: ">Hello\nworld",
+ blockquote: nil,
+ },
+ }
+
+ for _, test := range tests {
+ tokens := tokenizer.Tokenize(test.text)
+ require.Equal(t, test.blockquote, NewBlockquoteParser().Parse(tokens))
+ }
+}
diff --git a/plugin/gomark/parser/bold.go b/plugin/gomark/parser/bold.go
index 237cb8857..36cf24340 100644
--- a/plugin/gomark/parser/bold.go
+++ b/plugin/gomark/parser/bold.go
@@ -7,10 +7,8 @@ import (
type BoldParser struct{}
-var defaultBoldParser = &BoldParser{}
-
func NewBoldParser() InlineParser {
- return defaultBoldParser
+ return &BoldParser{}
}
func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) {
@@ -23,7 +21,7 @@ func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) {
return 0, false
}
prefixTokenType := prefixTokens[0].Type
- if prefixTokenType != tokenizer.Star && prefixTokenType != tokenizer.Underline {
+ if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {
return 0, false
}
diff --git a/plugin/gomark/parser/bold_italic.go b/plugin/gomark/parser/bold_italic.go
new file mode 100644
index 000000000..a97365942
--- /dev/null
+++ b/plugin/gomark/parser/bold_italic.go
@@ -0,0 +1,58 @@
+package parser
+
+import (
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+type BoldItalicParser struct{}
+
+func NewBoldItalicParser() InlineParser {
+ return &BoldItalicParser{}
+}
+
+func (*BoldItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {
+ if len(tokens) < 7 {
+ return 0, false
+ }
+
+ prefixTokens := tokens[:3]
+ if prefixTokens[0].Type != prefixTokens[1].Type || prefixTokens[0].Type != prefixTokens[2].Type || prefixTokens[1].Type != prefixTokens[2].Type {
+ return 0, false
+ }
+ prefixTokenType := prefixTokens[0].Type
+ if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {
+ return 0, false
+ }
+
+ cursor, matched := 3, false
+ for ; cursor < len(tokens)-2; cursor++ {
+ token, nextToken, endToken := tokens[cursor], tokens[cursor+1], tokens[cursor+2]
+ if token.Type == tokenizer.Newline || nextToken.Type == tokenizer.Newline || endToken.Type == tokenizer.Newline {
+ return 0, false
+ }
+ if token.Type == prefixTokenType && nextToken.Type == prefixTokenType && endToken.Type == prefixTokenType {
+ matched = true
+ break
+ }
+ }
+ if !matched {
+ return 0, false
+ }
+
+ return cursor + 3, true
+}
+
+func (p *BoldItalicParser) Parse(tokens []*tokenizer.Token) ast.Node {
+ size, ok := p.Match(tokens)
+ if size == 0 || !ok {
+ return nil
+ }
+
+ prefixTokenType := tokens[0].Type
+ contentTokens := tokens[3 : size-3]
+ return &ast.BoldItalic{
+ Symbol: prefixTokenType,
+ Content: tokenizer.Stringify(contentTokens),
+ }
+}
diff --git a/plugin/gomark/parser/bold_italic_test.go b/plugin/gomark/parser/bold_italic_test.go
new file mode 100644
index 000000000..855798593
--- /dev/null
+++ b/plugin/gomark/parser/bold_italic_test.go
@@ -0,0 +1,49 @@
+package parser
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+func TestBoldItalicParser(t *testing.T) {
+ tests := []struct {
+ text string
+ boldItalic ast.Node
+ }{
+ {
+ text: "*Hello world!",
+ boldItalic: nil,
+ },
+ {
+ text: "***Hello***",
+ boldItalic: &ast.BoldItalic{
+ Symbol: "*",
+ Content: "Hello",
+ },
+ },
+ {
+ text: "*** Hello ***",
+ boldItalic: &ast.BoldItalic{
+ Symbol: "*",
+ Content: " Hello ",
+ },
+ },
+ {
+ text: "*** Hello * *",
+ boldItalic: nil,
+ },
+ {
+ text: "*** Hello **",
+ boldItalic: nil,
+ },
+ }
+
+ for _, test := range tests {
+ tokens := tokenizer.Tokenize(test.text)
+ require.Equal(t, test.boldItalic, NewBoldItalicParser().Parse(tokens))
+ }
+}
diff --git a/plugin/gomark/parser/code.go b/plugin/gomark/parser/code.go
index 903b1cf77..b6dff80c6 100644
--- a/plugin/gomark/parser/code.go
+++ b/plugin/gomark/parser/code.go
@@ -7,10 +7,8 @@ import (
type CodeParser struct{}
-var defaultCodeParser = &CodeParser{}
-
func NewCodeParser() *CodeParser {
- return defaultCodeParser
+ return &CodeParser{}
}
func (*CodeParser) Match(tokens []*tokenizer.Token) (int, bool) {
diff --git a/plugin/gomark/parser/code_block.go b/plugin/gomark/parser/code_block.go
index 7cc8510fd..8ae48176f 100644
--- a/plugin/gomark/parser/code_block.go
+++ b/plugin/gomark/parser/code_block.go
@@ -10,10 +10,8 @@ type CodeBlockParser struct {
Content string
}
-var defaultCodeBlockParser = &CodeBlockParser{}
-
func NewCodeBlockParser() *CodeBlockParser {
- return defaultCodeBlockParser
+ return &CodeBlockParser{}
}
func (*CodeBlockParser) Match(tokens []*tokenizer.Token) (int, bool) {
diff --git a/plugin/gomark/parser/heading.go b/plugin/gomark/parser/heading.go
index 8becfc5f7..eaefcb333 100644
--- a/plugin/gomark/parser/heading.go
+++ b/plugin/gomark/parser/heading.go
@@ -62,11 +62,7 @@ func (p *HeadingParser) Parse(tokens []*tokenizer.Token) ast.Node {
}
}
contentTokens := tokens[level+1 : size]
- children := ParseInline(contentTokens, []InlineParser{
- NewBoldParser(),
- NewCodeParser(),
- NewTextParser(),
- })
+ children := ParseInline(contentTokens)
return &ast.Heading{
Level: level,
Children: children,
diff --git a/plugin/gomark/parser/horizontal_rule.go b/plugin/gomark/parser/horizontal_rule.go
new file mode 100644
index 000000000..a008d4433
--- /dev/null
+++ b/plugin/gomark/parser/horizontal_rule.go
@@ -0,0 +1,39 @@
+package parser
+
+import (
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+type HorizontalRuleParser struct{}
+
+func NewHorizontalRuleParser() *HorizontalRuleParser {
+ return &HorizontalRuleParser{}
+}
+
+func (*HorizontalRuleParser) Match(tokens []*tokenizer.Token) (int, bool) {
+ if len(tokens) < 3 {
+ return 0, false
+ }
+ if tokens[0].Type != tokens[1].Type || tokens[0].Type != tokens[2].Type || tokens[1].Type != tokens[2].Type {
+ return 0, false
+ }
+ if tokens[0].Type != tokenizer.Dash && tokens[0].Type != tokenizer.Underline && tokens[0].Type != tokenizer.Asterisk {
+ return 0, false
+ }
+ if len(tokens) > 3 && tokens[3].Type != tokenizer.Newline {
+ return 0, false
+ }
+ return 3, true
+}
+
+func (p *HorizontalRuleParser) Parse(tokens []*tokenizer.Token) ast.Node {
+ size, ok := p.Match(tokens)
+ if size == 0 || !ok {
+ return nil
+ }
+
+ return &ast.HorizontalRule{
+ Symbol: tokens[0].Type,
+ }
+}
diff --git a/plugin/gomark/parser/horizontal_rule_test.go b/plugin/gomark/parser/horizontal_rule_test.go
new file mode 100644
index 000000000..f45878f1a
--- /dev/null
+++ b/plugin/gomark/parser/horizontal_rule_test.go
@@ -0,0 +1,49 @@
+package parser
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+func TestHorizontalRuleParser(t *testing.T) {
+ tests := []struct {
+ text string
+ horizontalRule ast.Node
+ }{
+ {
+ text: "---",
+ horizontalRule: &ast.HorizontalRule{
+ Symbol: "-",
+ },
+ },
+ {
+ text: "****",
+ horizontalRule: nil,
+ },
+ {
+ text: "***",
+ horizontalRule: &ast.HorizontalRule{
+ Symbol: "*",
+ },
+ },
+ {
+ text: "-*-",
+ horizontalRule: nil,
+ },
+ {
+ text: "___",
+ horizontalRule: &ast.HorizontalRule{
+ Symbol: "_",
+ },
+ },
+ }
+
+ for _, test := range tests {
+ tokens := tokenizer.Tokenize(test.text)
+ require.Equal(t, test.horizontalRule, NewHorizontalRuleParser().Parse(tokens))
+ }
+}
diff --git a/plugin/gomark/parser/image.go b/plugin/gomark/parser/image.go
index 713981203..83be30bdf 100644
--- a/plugin/gomark/parser/image.go
+++ b/plugin/gomark/parser/image.go
@@ -7,10 +7,8 @@ import (
type ImageParser struct{}
-var defaultImageParser = &ImageParser{}
-
func NewImageParser() *ImageParser {
- return defaultImageParser
+ return &ImageParser{}
}
func (*ImageParser) Match(tokens []*tokenizer.Token) (int, bool) {
diff --git a/plugin/gomark/parser/italic.go b/plugin/gomark/parser/italic.go
index 3ab01f935..e9e32fefd 100644
--- a/plugin/gomark/parser/italic.go
+++ b/plugin/gomark/parser/italic.go
@@ -1,6 +1,9 @@
package parser
-import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+import (
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
type ItalicParser struct {
ContentTokens []*tokenizer.Token
@@ -10,21 +13,21 @@ func NewItalicParser() *ItalicParser {
return &ItalicParser{}
}
-func (*ItalicParser) Match(tokens []*tokenizer.Token) *ItalicParser {
+func (*ItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 3 {
- return nil
+ return 0, false
}
prefixTokens := tokens[:1]
- if prefixTokens[0].Type != tokenizer.Star && prefixTokens[0].Type != tokenizer.Underline {
- return nil
+ if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underline {
+ return 0, false
}
prefixTokenType := prefixTokens[0].Type
contentTokens := []*tokenizer.Token{}
matched := false
for _, token := range tokens[1:] {
if token.Type == tokenizer.Newline {
- return nil
+ return 0, false
}
if token.Type == prefixTokenType {
matched = true
@@ -33,10 +36,22 @@ func (*ItalicParser) Match(tokens []*tokenizer.Token) *ItalicParser {
contentTokens = append(contentTokens, token)
}
if !matched || len(contentTokens) == 0 {
+ return 0, false
+ }
+
+ return len(contentTokens) + 2, true
+}
+
+func (p *ItalicParser) Parse(tokens []*tokenizer.Token) ast.Node {
+ size, ok := p.Match(tokens)
+ if size == 0 || !ok {
return nil
}
- return &ItalicParser{
- ContentTokens: contentTokens,
+ prefixTokenType := tokens[0].Type
+ contentTokens := tokens[1 : size-1]
+ return &ast.Italic{
+ Symbol: prefixTokenType,
+ Content: tokenizer.Stringify(contentTokens),
}
}
diff --git a/plugin/gomark/parser/italic_test.go b/plugin/gomark/parser/italic_test.go
index 768931231..60c6ae534 100644
--- a/plugin/gomark/parser/italic_test.go
+++ b/plugin/gomark/parser/italic_test.go
@@ -5,13 +5,14 @@ import (
"github.com/stretchr/testify/require"
+ "github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestItalicParser(t *testing.T) {
tests := []struct {
text string
- italic *ItalicParser
+ italic ast.Node
}{
{
text: "*Hello world!",
@@ -19,76 +20,29 @@ func TestItalicParser(t *testing.T) {
},
{
text: "*Hello*",
- italic: &ItalicParser{
- ContentTokens: []*tokenizer.Token{
- {
- Type: tokenizer.Text,
- Value: "Hello",
- },
- },
+ italic: &ast.Italic{
+ Symbol: "*",
+ Content: "Hello",
},
},
{
text: "* Hello *",
- italic: &ItalicParser{
- ContentTokens: []*tokenizer.Token{
- {
- Type: tokenizer.Space,
- Value: " ",
- },
- {
- Type: tokenizer.Text,
- Value: "Hello",
- },
- {
- Type: tokenizer.Space,
- Value: " ",
- },
- },
+ italic: &ast.Italic{
+ Symbol: "*",
+ Content: " Hello ",
},
},
- {
- text: "** Hello * *",
- italic: nil,
- },
{
text: "*1* Hello * *",
- italic: &ItalicParser{
- ContentTokens: []*tokenizer.Token{
- {
- Type: tokenizer.Text,
- Value: "1",
- },
- },
+ italic: &ast.Italic{
+ Symbol: "*",
+ Content: "1",
},
},
- {
- text: `* \n * Hello * *`,
- italic: &ItalicParser{
- ContentTokens: []*tokenizer.Token{
- {
- Type: tokenizer.Space,
- Value: " ",
- },
- {
- Type: tokenizer.Text,
- Value: `\n`,
- },
- {
- Type: tokenizer.Space,
- Value: " ",
- },
- },
- },
- },
- {
- text: "* \n * Hello * *",
- italic: nil,
- },
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
- require.Equal(t, test.italic, NewItalicParser().Match(tokens))
+ require.Equal(t, test.italic, NewItalicParser().Parse(tokens))
}
}
diff --git a/plugin/gomark/parser/line_break.go b/plugin/gomark/parser/line_break.go
index bdc0957d6..6017f72a7 100644
--- a/plugin/gomark/parser/line_break.go
+++ b/plugin/gomark/parser/line_break.go
@@ -7,10 +7,8 @@ import (
type LineBreakParser struct{}
-var defaultLineBreakParser = &LineBreakParser{}
-
func NewLineBreakParser() *LineBreakParser {
- return defaultLineBreakParser
+ return &LineBreakParser{}
}
func (*LineBreakParser) Match(tokens []*tokenizer.Token) (int, bool) {
diff --git a/plugin/gomark/parser/link.go b/plugin/gomark/parser/link.go
index a1c8b7449..3b5fd6439 100644
--- a/plugin/gomark/parser/link.go
+++ b/plugin/gomark/parser/link.go
@@ -1,58 +1,72 @@
package parser
-import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+import (
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
-type LinkParser struct {
- ContentTokens []*tokenizer.Token
- URL string
-}
+type LinkParser struct{}
func NewLinkParser() *LinkParser {
return &LinkParser{}
}
-func (*LinkParser) Match(tokens []*tokenizer.Token) *LinkParser {
- if len(tokens) < 4 {
- return nil
+func (*LinkParser) Match(tokens []*tokenizer.Token) (int, bool) {
+ if len(tokens) < 5 {
+ return 0, false
}
if tokens[0].Type != tokenizer.LeftSquareBracket {
- return nil
+ return 0, false
}
- cursor, contentTokens := 1, []*tokenizer.Token{}
- for ; cursor < len(tokens)-2; cursor++ {
- if tokens[cursor].Type == tokenizer.Newline {
- return nil
+ textTokens := []*tokenizer.Token{}
+ for _, token := range tokens[1:] {
+ if token.Type == tokenizer.Newline {
+ return 0, false
}
- if tokens[cursor].Type == tokenizer.RightSquareBracket {
+ if token.Type == tokenizer.RightSquareBracket {
break
}
- contentTokens = append(contentTokens, tokens[cursor])
+ textTokens = append(textTokens, token)
}
- if tokens[cursor+1].Type != tokenizer.LeftParenthesis {
- return nil
+ if len(textTokens)+4 >= len(tokens) {
+ return 0, false
+ }
+ if tokens[2+len(textTokens)].Type != tokenizer.LeftParenthesis {
+ return 0, false
}
- matched, url := false, ""
- for _, token := range tokens[cursor+2:] {
+ urlTokens := []*tokenizer.Token{}
+ for _, token := range tokens[3+len(textTokens):] {
if token.Type == tokenizer.Newline || token.Type == tokenizer.Space {
- return nil
+ return 0, false
}
if token.Type == tokenizer.RightParenthesis {
- matched = true
break
}
- url += token.Value
+ urlTokens = append(urlTokens, token)
+ }
+ if 4+len(urlTokens)+len(textTokens) > len(tokens) {
+ return 0, false
}
- if !matched || url == "" {
+
+ return 4 + len(urlTokens) + len(textTokens), true
+}
+
+func (p *LinkParser) Parse(tokens []*tokenizer.Token) ast.Node {
+ size, ok := p.Match(tokens)
+ if size == 0 || !ok {
return nil
}
- if len(contentTokens) == 0 {
- contentTokens = append(contentTokens, &tokenizer.Token{
- Type: tokenizer.Text,
- Value: url,
- })
+
+ textTokens := []*tokenizer.Token{}
+ for _, token := range tokens[1:] {
+ if token.Type == tokenizer.RightSquareBracket {
+ break
+ }
+ textTokens = append(textTokens, token)
}
- return &LinkParser{
- ContentTokens: contentTokens,
- URL: url,
+ urlTokens := tokens[2+len(textTokens)+1 : size-1]
+ return &ast.Link{
+ Text: tokenizer.Stringify(textTokens),
+ URL: tokenizer.Stringify(urlTokens),
}
}
diff --git a/plugin/gomark/parser/link_test.go b/plugin/gomark/parser/link_test.go
index ab4f5c7dc..421ab38a5 100644
--- a/plugin/gomark/parser/link_test.go
+++ b/plugin/gomark/parser/link_test.go
@@ -5,24 +5,20 @@ import (
"github.com/stretchr/testify/require"
+ "github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestLinkParser(t *testing.T) {
tests := []struct {
text string
- link *LinkParser
+ link ast.Node
}{
{
text: "[](https://example.com)",
- link: &LinkParser{
- ContentTokens: []*tokenizer.Token{
- {
- Type: tokenizer.Text,
- Value: "https://example.com",
- },
- },
- URL: "https://example.com",
+ link: &ast.Link{
+ Text: "",
+ URL: "https://example.com",
},
},
{
@@ -35,27 +31,14 @@ func TestLinkParser(t *testing.T) {
},
{
text: "[hello world](https://example.com)",
- link: &LinkParser{
- ContentTokens: []*tokenizer.Token{
- {
- Type: tokenizer.Text,
- Value: "hello",
- },
- {
- Type: tokenizer.Space,
- Value: " ",
- },
- {
- Type: tokenizer.Text,
- Value: "world",
- },
- },
- URL: "https://example.com",
+ link: &ast.Link{
+ Text: "hello world",
+ URL: "https://example.com",
},
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
- require.Equal(t, test.link, NewLinkParser().Match(tokens))
+ require.Equal(t, test.link, NewLinkParser().Parse(tokens))
}
}
diff --git a/plugin/gomark/parser/paragraph.go b/plugin/gomark/parser/paragraph.go
index c2403ad02..09e69047f 100644
--- a/plugin/gomark/parser/paragraph.go
+++ b/plugin/gomark/parser/paragraph.go
@@ -9,10 +9,8 @@ type ParagraphParser struct {
ContentTokens []*tokenizer.Token
}
-var defaultParagraphParser = &ParagraphParser{}
-
func NewParagraphParser() *ParagraphParser {
- return defaultParagraphParser
+ return &ParagraphParser{}
}
func (*ParagraphParser) Match(tokens []*tokenizer.Token) (int, bool) {
@@ -38,10 +36,7 @@ func (p *ParagraphParser) Parse(tokens []*tokenizer.Token) ast.Node {
}
contentTokens := tokens[:size]
- children := ParseInline(contentTokens, []InlineParser{
- NewBoldParser(),
- NewTextParser(),
- })
+ children := ParseInline(contentTokens)
return &ast.Paragraph{
Children: children,
}
diff --git a/plugin/gomark/parser/parser.go b/plugin/gomark/parser/parser.go
index c82fa50d5..123eead34 100644
--- a/plugin/gomark/parser/parser.go
+++ b/plugin/gomark/parser/parser.go
@@ -1,6 +1,8 @@
package parser
import (
+ "errors"
+
"github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
@@ -23,32 +25,51 @@ type BlockParser interface {
BaseParser
}
-func Parse(tokens []*tokenizer.Token) []ast.Node {
+var defaultBlockParsers = []BlockParser{
+ NewCodeBlockParser(),
+ NewHorizontalRuleParser(),
+ NewHeadingParser(),
+ NewBlockquoteParser(),
+ NewParagraphParser(),
+ NewLineBreakParser(),
+}
+
+func Parse(tokens []*tokenizer.Token) ([]ast.Node, error) {
nodes := []ast.Node{}
- blockParsers := []BlockParser{
- NewCodeBlockParser(),
- NewParagraphParser(),
- NewLineBreakParser(),
- }
for len(tokens) > 0 {
- for _, blockParser := range blockParsers {
+ for _, blockParser := range defaultBlockParsers {
cursor, matched := blockParser.Match(tokens)
if matched {
node := blockParser.Parse(tokens)
+ if node == nil {
+ return nil, errors.New("parse error")
+ }
nodes = append(nodes, node)
tokens = tokens[cursor:]
break
}
}
}
- return nodes
+ return nodes, nil
+}
+
+var defaultInlineParsers = []InlineParser{
+ NewBoldItalicParser(),
+ NewImageParser(),
+ NewLinkParser(),
+ NewBoldParser(),
+ NewItalicParser(),
+ NewCodeParser(),
+ NewTagParser(),
+ NewStrikethroughParser(),
+ NewTextParser(),
}
-func ParseInline(tokens []*tokenizer.Token, inlineParsers []InlineParser) []ast.Node {
+func ParseInline(tokens []*tokenizer.Token) []ast.Node {
nodes := []ast.Node{}
var lastNode ast.Node
for len(tokens) > 0 {
- for _, inlineParser := range inlineParsers {
+ for _, inlineParser := range defaultInlineParsers {
cursor, matched := inlineParser.Match(tokens)
if matched {
node := inlineParser.Parse(tokens)
diff --git a/plugin/gomark/parser/parser_test.go b/plugin/gomark/parser/parser_test.go
index dc2d6e68a..62b06ab90 100644
--- a/plugin/gomark/parser/parser_test.go
+++ b/plugin/gomark/parser/parser_test.go
@@ -89,6 +89,8 @@ func TestParser(t *testing.T) {
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
- require.Equal(t, test.nodes, Parse(tokens))
+ nodes, err := Parse(tokens)
+ require.NoError(t, err)
+ require.Equal(t, test.nodes, nodes)
}
}
diff --git a/plugin/gomark/parser/strikethrough.go b/plugin/gomark/parser/strikethrough.go
new file mode 100644
index 000000000..fde8acc59
--- /dev/null
+++ b/plugin/gomark/parser/strikethrough.go
@@ -0,0 +1,49 @@
+package parser
+
+import (
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+type StrikethroughParser struct{}
+
+func NewStrikethroughParser() *StrikethroughParser {
+ return &StrikethroughParser{}
+}
+
+func (*StrikethroughParser) Match(tokens []*tokenizer.Token) (int, bool) {
+ if len(tokens) < 5 {
+ return 0, false
+ }
+ if tokens[0].Type != tokenizer.Tilde || tokens[1].Type != tokenizer.Tilde {
+ return 0, false
+ }
+
+ cursor, matched := 2, false
+ for ; cursor < len(tokens)-1; cursor++ {
+ token, nextToken := tokens[cursor], tokens[cursor+1]
+ if token.Type == tokenizer.Newline || nextToken.Type == tokenizer.Newline {
+ return 0, false
+ }
+ if token.Type == tokenizer.Tilde && nextToken.Type == tokenizer.Tilde {
+ matched = true
+ break
+ }
+ }
+ if !matched {
+ return 0, false
+ }
+ return cursor + 2, true
+}
+
+func (p *StrikethroughParser) Parse(tokens []*tokenizer.Token) ast.Node {
+ size, ok := p.Match(tokens)
+ if size == 0 || !ok {
+ return nil
+ }
+
+ contentTokens := tokens[2 : size-2]
+ return &ast.Strikethrough{
+ Content: tokenizer.Stringify(contentTokens),
+ }
+}
diff --git a/plugin/gomark/parser/strikethrough_test.go b/plugin/gomark/parser/strikethrough_test.go
new file mode 100644
index 000000000..28a04bcc6
--- /dev/null
+++ b/plugin/gomark/parser/strikethrough_test.go
@@ -0,0 +1,45 @@
+package parser
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+func TestStrikethroughParser(t *testing.T) {
+ tests := []struct {
+ text string
+ strikethrough ast.Node
+ }{
+ {
+ text: "~~Hello world",
+ strikethrough: nil,
+ },
+ {
+ text: "~~Hello~~",
+ strikethrough: &ast.Strikethrough{
+ Content: "Hello",
+ },
+ },
+ {
+ text: "~~ Hello ~~",
+ strikethrough: &ast.Strikethrough{
+ Content: " Hello ",
+ },
+ },
+ {
+ text: "~~1~~ Hello ~~~",
+ strikethrough: &ast.Strikethrough{
+ Content: "1",
+ },
+ },
+ }
+
+ for _, test := range tests {
+ tokens := tokenizer.Tokenize(test.text)
+ require.Equal(t, test.strikethrough, NewStrikethroughParser().Parse(tokens))
+ }
+}
diff --git a/plugin/gomark/parser/tag.go b/plugin/gomark/parser/tag.go
index c33fcd0b4..80fc15ed4 100644
--- a/plugin/gomark/parser/tag.go
+++ b/plugin/gomark/parser/tag.go
@@ -1,21 +1,22 @@
package parser
-import "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+import (
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
-type TagParser struct {
- ContentTokens []*tokenizer.Token
-}
+type TagParser struct{}
func NewTagParser() *TagParser {
return &TagParser{}
}
-func (*TagParser) Match(tokens []*tokenizer.Token) *TagParser {
+func (*TagParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 2 {
- return nil
+ return 0, false
}
if tokens[0].Type != tokenizer.Hash {
- return nil
+ return 0, false
}
contentTokens := []*tokenizer.Token{}
for _, token := range tokens[1:] {
@@ -25,10 +26,20 @@ func (*TagParser) Match(tokens []*tokenizer.Token) *TagParser {
contentTokens = append(contentTokens, token)
}
if len(contentTokens) == 0 {
+ return 0, false
+ }
+
+ return len(contentTokens) + 1, true
+}
+
+func (p *TagParser) Parse(tokens []*tokenizer.Token) ast.Node {
+ size, ok := p.Match(tokens)
+ if size == 0 || !ok {
return nil
}
- return &TagParser{
- ContentTokens: contentTokens,
+ contentTokens := tokens[1:size]
+ return &ast.Tag{
+ Content: tokenizer.Stringify(contentTokens),
}
}
diff --git a/plugin/gomark/parser/tag_test.go b/plugin/gomark/parser/tag_test.go
index 7066027f5..54d541e71 100644
--- a/plugin/gomark/parser/tag_test.go
+++ b/plugin/gomark/parser/tag_test.go
@@ -5,13 +5,14 @@ import (
"github.com/stretchr/testify/require"
+ "github.com/usememos/memos/plugin/gomark/ast"
"github.com/usememos/memos/plugin/gomark/parser/tokenizer"
)
func TestTagParser(t *testing.T) {
tests := []struct {
text string
- tag *TagParser
+ tag ast.Node
}{
{
text: "*Hello world",
@@ -23,30 +24,20 @@ func TestTagParser(t *testing.T) {
},
{
text: "#tag",
- tag: &TagParser{
- ContentTokens: []*tokenizer.Token{
- {
- Type: tokenizer.Text,
- Value: "tag",
- },
- },
+ tag: &ast.Tag{
+ Content: "tag",
},
},
{
- text: "#tag/subtag",
- tag: &TagParser{
- ContentTokens: []*tokenizer.Token{
- {
- Type: tokenizer.Text,
- Value: "tag/subtag",
- },
- },
+ text: "#tag/subtag 123",
+ tag: &ast.Tag{
+ Content: "tag/subtag",
},
},
}
for _, test := range tests {
tokens := tokenizer.Tokenize(test.text)
- require.Equal(t, test.tag, NewTagParser().Match(tokens))
+ require.Equal(t, test.tag, NewTagParser().Parse(tokens))
}
}
diff --git a/plugin/gomark/parser/text.go b/plugin/gomark/parser/text.go
index f930ad3dd..a2523ff29 100644
--- a/plugin/gomark/parser/text.go
+++ b/plugin/gomark/parser/text.go
@@ -9,10 +9,8 @@ type TextParser struct {
Content string
}
-var defaultTextParser = &TextParser{}
-
func NewTextParser() *TextParser {
- return defaultTextParser
+ return &TextParser{}
}
func (*TextParser) Match(tokens []*tokenizer.Token) (int, bool) {
diff --git a/plugin/gomark/parser/tokenizer/tokenizer.go b/plugin/gomark/parser/tokenizer/tokenizer.go
index a9c9f42ce..f27ecf03b 100644
--- a/plugin/gomark/parser/tokenizer/tokenizer.go
+++ b/plugin/gomark/parser/tokenizer/tokenizer.go
@@ -4,7 +4,7 @@ type TokenType = string
const (
Underline TokenType = "_"
- Star TokenType = "*"
+ Asterisk TokenType = "*"
Hash TokenType = "#"
Backtick TokenType = "`"
LeftSquareBracket TokenType = "["
@@ -12,6 +12,9 @@ const (
LeftParenthesis TokenType = "("
RightParenthesis TokenType = ")"
ExclamationMark TokenType = "!"
+ Tilde TokenType = "~"
+ Dash TokenType = "-"
+ GreaterThan TokenType = ">"
Newline TokenType = "\n"
Space TokenType = " "
)
@@ -39,7 +42,7 @@ func Tokenize(text string) []*Token {
case '_':
tokens = append(tokens, NewToken(Underline, "_"))
case '*':
- tokens = append(tokens, NewToken(Star, "*"))
+ tokens = append(tokens, NewToken(Asterisk, "*"))
case '#':
tokens = append(tokens, NewToken(Hash, "#"))
case '`':
@@ -54,6 +57,12 @@ func Tokenize(text string) []*Token {
tokens = append(tokens, NewToken(RightParenthesis, ")"))
case '!':
tokens = append(tokens, NewToken(ExclamationMark, "!"))
+ case '~':
+ tokens = append(tokens, NewToken(Tilde, "~"))
+ case '-':
+ tokens = append(tokens, NewToken(Dash, "-"))
+ case '>':
+ tokens = append(tokens, NewToken(GreaterThan, ">"))
case '\n':
tokens = append(tokens, NewToken(Newline, "\n"))
case ' ':
diff --git a/plugin/gomark/parser/tokenizer/tokenizer_test.go b/plugin/gomark/parser/tokenizer/tokenizer_test.go
index 8010fe4f6..a85651b3d 100644
--- a/plugin/gomark/parser/tokenizer/tokenizer_test.go
+++ b/plugin/gomark/parser/tokenizer/tokenizer_test.go
@@ -15,7 +15,7 @@ func TestTokenize(t *testing.T) {
text: "*Hello world!",
tokens: []*Token{
{
- Type: Star,
+ Type: Asterisk,
Value: "*",
},
{