Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
chore: implement list nodes
Stevenb00443c22263
diff --git a/plugin/gomark/ast/block.go b/plugin/gomark/ast/block.goindex b5390f3ea..c69d6a6a9 100644--- a/plugin/gomark/ast/block.go+++ b/plugin/gomark/ast/block.go@@ -64,3 +64,26 @@ type Blockquote struct {func (*Blockquote) Type() NodeType {return BlockquoteNode}++type OrderedList struct {+ BaseBlock++ Number string+ Children []Node+}++func (*OrderedList) Type() NodeType {+ return OrderedListNode+}++type UnorderedList struct {+ BaseBlock++ // Symbol is "*" or "-" or "+".+ Symbol string+ Children []Node+}++func (*UnorderedList) Type() NodeType {+ return UnorderedListNode+}diff --git a/plugin/gomark/ast/node.go b/plugin/gomark/ast/node.goindex 6fbb23b58..9e4c53150 100644--- a/plugin/gomark/ast/node.go+++ b/plugin/gomark/ast/node.go@@ -11,6 +11,8 @@ const (HeadingNodeHorizontalRuleNodeBlockquoteNode+ OrderedListNode+ UnorderedListNode// Inline nodes.TextNodeBoldNodediff --git a/plugin/gomark/parser/bold.go b/plugin/gomark/parser/bold.goindex 530b8a995..f1c1603f8 100644--- a/plugin/gomark/parser/bold.go+++ b/plugin/gomark/parser/bold.go@@ -23,7 +23,7 @@ func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) {return 0, false}prefixTokenType := prefixTokens[0].Type- if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {+ if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore {return 0, false}diff --git a/plugin/gomark/parser/bold_italic.go b/plugin/gomark/parser/bold_italic.goindex 44f43a3e8..5c43bab9b 100644--- a/plugin/gomark/parser/bold_italic.go+++ b/plugin/gomark/parser/bold_italic.go@@ -23,7 +23,7 @@ func (*BoldItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {return 0, false}prefixTokenType := prefixTokens[0].Type- if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {+ if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore {return 0, false}diff --git a/plugin/gomark/parser/heading.go b/plugin/gomark/parser/heading.goindex 284c88820..bd79f146b 100644--- a/plugin/gomark/parser/heading.go+++ b/plugin/gomark/parser/heading.go@@ -16,7 +16,7 @@ func NewHeadingParser() *HeadingParser {func (*HeadingParser) Match(tokens []*tokenizer.Token) (int, bool) {cursor := 0for _, token := range tokens {- if token.Type == tokenizer.Hash {+ if token.Type == tokenizer.PoundSign {cursor++} else {break@@ -57,7 +57,7 @@ func (p *HeadingParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {level := 0for _, token := range tokens {- if token.Type == tokenizer.Hash {+ if token.Type == tokenizer.PoundSign {level++} else {breakdiff --git a/plugin/gomark/parser/horizontal_rule.go b/plugin/gomark/parser/horizontal_rule.goindex a3db6d1d4..75d6c50a2 100644--- a/plugin/gomark/parser/horizontal_rule.go+++ b/plugin/gomark/parser/horizontal_rule.go@@ -20,7 +20,7 @@ func (*HorizontalRuleParser) Match(tokens []*tokenizer.Token) (int, bool) {if tokens[0].Type != tokens[1].Type || tokens[0].Type != tokens[2].Type || tokens[1].Type != tokens[2].Type {return 0, false}- if tokens[0].Type != tokenizer.Dash && tokens[0].Type != tokenizer.Underline && tokens[0].Type != tokenizer.Asterisk {+ if tokens[0].Type != tokenizer.Hyphen && tokens[0].Type != tokenizer.Underscore && tokens[0].Type != tokenizer.Asterisk {return 0, false}if len(tokens) > 3 && tokens[3].Type != tokenizer.Newline {diff --git a/plugin/gomark/parser/italic.go b/plugin/gomark/parser/italic.goindex 1b29afeb0..6a80496c1 100644--- a/plugin/gomark/parser/italic.go+++ b/plugin/gomark/parser/italic.go@@ -21,7 +21,7 @@ func (*ItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {}prefixTokens := tokens[:1]- if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underline {+ if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underscore {return 0, false}prefixTokenType := prefixTokens[0].Typediff --git a/plugin/gomark/parser/ordered_list.go b/plugin/gomark/parser/ordered_list.gonew file mode 100644index 000000000..4e01b4760--- /dev/null+++ b/plugin/gomark/parser/ordered_list.go@@ -0,0 +1,54 @@+package parser++import (+ "errors"++ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++type OrderedListParser struct{}++func NewOrderedListParser() *OrderedListParser {+ return &OrderedListParser{}+}++func (*OrderedListParser) Match(tokens []*tokenizer.Token) (int, bool) {+ if len(tokens) < 4 {+ return 0, false+ }+ if tokens[0].Type != tokenizer.Number || tokens[1].Type != tokenizer.Dot || tokens[2].Type != tokenizer.Space {+ return 0, false+ }++ contentTokens := []*tokenizer.Token{}+ for _, token := range tokens[3:] {+ contentTokens = append(contentTokens, token)+ if token.Type == tokenizer.Newline {+ break+ }+ }++ if len(contentTokens) == 0 {+ return 0, false+ }++ return len(contentTokens) + 3, true+}++func (p *OrderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {+ size, ok := p.Match(tokens)+ if size == 0 || !ok {+ return nil, errors.New("not matched")+ }++ contentTokens := tokens[3:size]+ children, err := ParseInline(contentTokens)+ if err != nil {+ return nil, err+ }+ return &ast.OrderedList{+ Number: tokens[0].Value,+ Children: children,+ }, nil+}diff --git a/plugin/gomark/parser/ordered_list_test.go b/plugin/gomark/parser/ordered_list_test.gonew file mode 100644index 000000000..374c317af--- /dev/null+++ b/plugin/gomark/parser/ordered_list_test.go@@ -0,0 +1,58 @@+package parser++import (+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++func TestOrderedListParser(t *testing.T) {+ tests := []struct {+ text string+ node ast.Node+ }{+ {+ text: "1.asd",+ node: nil,+ },+ {+ text: "1. Hello World",+ node: &ast.OrderedList{+ Number: "1",+ Children: []ast.Node{+ &ast.Text{+ Content: "Hello World",+ },+ },+ },+ },+ {+ text: "1aa. Hello World",+ node: nil,+ },+ {+ text: "22. Hello *World*",+ node: &ast.OrderedList{+ Number: "22",+ Children: []ast.Node{+ &ast.Text{+ Content: "Hello ",+ },+ &ast.Italic{+ Symbol: "*",+ Content: "World",+ },+ },+ },+ },+ }++ for _, test := range tests {+ tokens := tokenizer.Tokenize(test.text)+ node, _ := NewOrderedListParser().Parse(tokens)+ require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node}))+ }+}diff --git a/plugin/gomark/parser/tag.go b/plugin/gomark/parser/tag.goindex 3deccad39..9a7d872f1 100644--- a/plugin/gomark/parser/tag.go+++ b/plugin/gomark/parser/tag.go@@ -17,12 +17,12 @@ func (*TagParser) Match(tokens []*tokenizer.Token) (int, bool) {if len(tokens) < 2 {return 0, false}- if tokens[0].Type != tokenizer.Hash {+ if tokens[0].Type != tokenizer.PoundSign {return 0, false}contentTokens := []*tokenizer.Token{}for _, token := range tokens[1:] {- if token.Type == tokenizer.Newline || token.Type == tokenizer.Space || token.Type == tokenizer.Hash {+ if token.Type == tokenizer.Newline || token.Type == tokenizer.Space || token.Type == tokenizer.PoundSign {break}contentTokens = append(contentTokens, token)diff --git a/plugin/gomark/parser/tokenizer/tokenizer.go b/plugin/gomark/parser/tokenizer/tokenizer.goindex ce3730014..e138ba829 100644--- a/plugin/gomark/parser/tokenizer/tokenizer.go+++ b/plugin/gomark/parser/tokenizer/tokenizer.go@@ -3,9 +3,9 @@ package tokenizertype TokenType = stringconst (- Underline TokenType = "_"+ Underscore TokenType = "_"Asterisk TokenType = "*"- Hash TokenType = "#"+ PoundSign TokenType = "#"Backtick TokenType = "`"LeftSquareBracket TokenType = "["RightSquareBracket TokenType = "]"@@ -13,14 +13,17 @@ const (RightParenthesis TokenType = ")"ExclamationMark TokenType = "!"Tilde TokenType = "~"- Dash TokenType = "-"+ Hyphen TokenType = "-"+ PlusSign TokenType = "+"+ Dot TokenType = "."GreaterThan TokenType = ">"Newline TokenType = "\n"Space TokenType = " ")const (- Text TokenType = ""+ Number TokenType = "number"+ Text TokenType = "")type Token struct {@@ -40,11 +43,11 @@ func Tokenize(text string) []*Token {for _, c := range text {switch c {case '_':- tokens = append(tokens, NewToken(Underline, "_"))+ tokens = append(tokens, NewToken(Underscore, "_"))case '*':tokens = append(tokens, NewToken(Asterisk, "*"))case '#':- tokens = append(tokens, NewToken(Hash, "#"))+ tokens = append(tokens, NewToken(PoundSign, "#"))case '`':tokens = append(tokens, NewToken(Backtick, "`"))case '[':@@ -60,9 +63,13 @@ func Tokenize(text string) []*Token {case '~':tokens = append(tokens, NewToken(Tilde, "~"))case '-':- tokens = append(tokens, NewToken(Dash, "-"))+ tokens = append(tokens, NewToken(Hyphen, "-"))case '>':tokens = append(tokens, NewToken(GreaterThan, ">"))+ case '+':+ tokens = append(tokens, NewToken(PlusSign, "+"))+ case '.':+ tokens = append(tokens, NewToken(Dot, "."))case '\n':tokens = append(tokens, NewToken(Newline, "\n"))case ' ':@@ -72,10 +79,19 @@ func Tokenize(text string) []*Token {if len(tokens) > 0 {prevToken = tokens[len(tokens)-1]}- if prevToken == nil || prevToken.Type != Text {- tokens = append(tokens, NewToken(Text, string(c)))++ isNumber := c >= '0' && c <= '9'+ if prevToken != nil {+ if (prevToken.Type == Text && !isNumber) || (prevToken.Type == Number && isNumber) {+ prevToken.Value += string(c)+ continue+ }+ }++ if isNumber {+ tokens = append(tokens, NewToken(Number, string(c)))} else {- prevToken.Value += string(c)+ tokens = append(tokens, NewToken(Text, string(c)))}}}diff --git a/plugin/gomark/parser/tokenizer/tokenizer_test.go b/plugin/gomark/parser/tokenizer/tokenizer_test.goindex a85651b3d..284dba939 100644--- a/plugin/gomark/parser/tokenizer/tokenizer_test.go+++ b/plugin/gomark/parser/tokenizer/tokenizer_test.go@@ -41,7 +41,7 @@ func TestTokenize(t *testing.T) {world`,tokens: []*Token{{- Type: Hash,+ Type: PoundSign,Value: "#",},{diff --git a/plugin/gomark/parser/unordered_list.go b/plugin/gomark/parser/unordered_list.gonew file mode 100644index 000000000..c64daeb17--- /dev/null+++ b/plugin/gomark/parser/unordered_list.go@@ -0,0 +1,55 @@+package parser++import (+ "errors"++ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++type UnorderedListParser struct{}++func NewUnorderedListParser() *UnorderedListParser {+ return &UnorderedListParser{}+}++func (*UnorderedListParser) Match(tokens []*tokenizer.Token) (int, bool) {+ if len(tokens) < 3 {+ return 0, false+ }+ symbolToken := tokens[0]+ if (symbolToken.Type != tokenizer.Hyphen && symbolToken.Type != tokenizer.Asterisk && symbolToken.Type != tokenizer.PlusSign) || tokens[1].Type != tokenizer.Space {+ return 0, false+ }++ contentTokens := []*tokenizer.Token{}+ for _, token := range tokens[2:] {+ if token.Type == tokenizer.Newline {+ break+ }+ contentTokens = append(contentTokens, token)+ }+ if len(contentTokens) == 0 {+ return 0, false+ }++ return len(contentTokens) + 2, true+}++func (p *UnorderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {+ size, ok := p.Match(tokens)+ if size == 0 || !ok {+ return nil, errors.New("not matched")+ }++ symbolToken := tokens[0]+ contentTokens := tokens[2:size]+ children, err := ParseInline(contentTokens)+ if err != nil {+ return nil, err+ }+ return &ast.UnorderedList{+ Symbol: symbolToken.Type,+ Children: children,+ }, nil+}diff --git a/plugin/gomark/parser/unordered_list_test.go b/plugin/gomark/parser/unordered_list_test.gonew file mode 100644index 000000000..982d58be7--- /dev/null+++ b/plugin/gomark/parser/unordered_list_test.go@@ -0,0 +1,51 @@+package parser++import (+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/usememos/memos/plugin/gomark/ast"+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"+)++func TestUnorderedListParser(t *testing.T) {+ tests := []struct {+ text string+ node ast.Node+ }{+ {+ text: "*asd",+ node: nil,+ },+ {+ text: "+ Hello World",+ node: &ast.UnorderedList{+ Symbol: tokenizer.PlusSign,+ Children: []ast.Node{+ &ast.Text{+ Content: "Hello World",+ },+ },+ },+ },+ {+ text: "* **Hello**",+ node: &ast.UnorderedList{+ Symbol: tokenizer.Asterisk,+ Children: []ast.Node{+ &ast.Bold{+ Symbol: "*",+ Content: "Hello",+ },+ },+ },+ },+ }++ for _, test := range tests {+ tokens := tokenizer.Tokenize(test.text)+ node, _ := NewUnorderedListParser().Parse(tokens)+ require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node}))+ }+}diff --git a/plugin/gomark/render/html/html.go b/plugin/gomark/render/html/html.goindex aedbd222b..46f90bc66 100644--- a/plugin/gomark/render/html/html.go+++ b/plugin/gomark/render/html/html.go@@ -159,7 +159,7 @@ func (r *HTMLRender) renderLink(node *ast.Link) {func (r *HTMLRender) renderTag(node *ast.Tag) {r.output.WriteString(`<span>`)- r.output.WriteString(`# `)+ r.output.WriteString(`#`)r.output.WriteString(node.Content)r.output.WriteString(`</span>`)}diff --git a/plugin/gomark/render/html/html_test.go b/plugin/gomark/render/html/html_test.goindex 9c22097ac..25164254d 100644--- a/plugin/gomark/render/html/html_test.go+++ b/plugin/gomark/render/html/html_test.go@@ -30,6 +30,10 @@ func TestHTMLRender(t *testing.T) {text: "**Hello** world!",expected: `<p><strong>Hello</strong> world!</p>`,},+ {+ text: "#article #memo",+ expected: `<p><span>#article</span> <span>#memo</span></p>`,+ },}for _, test := range tests {