Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

chore: implement list nodes

Stevenb00443c22263
diff --git a/plugin/gomark/ast/block.go b/plugin/gomark/ast/block.go
index b5390f3ea..c69d6a6a9 100644
--- a/plugin/gomark/ast/block.go
+++ b/plugin/gomark/ast/block.go
@@ -64,3 +64,26 @@ type Blockquote struct {
func (*Blockquote) Type() NodeType {
return BlockquoteNode
}
+
+type OrderedList struct {
+ BaseBlock
+
+ Number string
+ Children []Node
+}
+
+func (*OrderedList) Type() NodeType {
+ return OrderedListNode
+}
+
+type UnorderedList struct {
+ BaseBlock
+
+ // Symbol is "*" or "-" or "+".
+ Symbol string
+ Children []Node
+}
+
+func (*UnorderedList) Type() NodeType {
+ return UnorderedListNode
+}
diff --git a/plugin/gomark/ast/node.go b/plugin/gomark/ast/node.go
index 6fbb23b58..9e4c53150 100644
--- a/plugin/gomark/ast/node.go
+++ b/plugin/gomark/ast/node.go
@@ -11,6 +11,8 @@ const (
HeadingNode
HorizontalRuleNode
BlockquoteNode
+ OrderedListNode
+ UnorderedListNode
// Inline nodes.
TextNode
BoldNode
diff --git a/plugin/gomark/parser/bold.go b/plugin/gomark/parser/bold.go
index 530b8a995..f1c1603f8 100644
--- a/plugin/gomark/parser/bold.go
+++ b/plugin/gomark/parser/bold.go
@@ -23,7 +23,7 @@ func (*BoldParser) Match(tokens []*tokenizer.Token) (int, bool) {
return 0, false
}
prefixTokenType := prefixTokens[0].Type
- if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {
+ if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore {
return 0, false
}
diff --git a/plugin/gomark/parser/bold_italic.go b/plugin/gomark/parser/bold_italic.go
index 44f43a3e8..5c43bab9b 100644
--- a/plugin/gomark/parser/bold_italic.go
+++ b/plugin/gomark/parser/bold_italic.go
@@ -23,7 +23,7 @@ func (*BoldItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {
return 0, false
}
prefixTokenType := prefixTokens[0].Type
- if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underline {
+ if prefixTokenType != tokenizer.Asterisk && prefixTokenType != tokenizer.Underscore {
return 0, false
}
diff --git a/plugin/gomark/parser/heading.go b/plugin/gomark/parser/heading.go
index 284c88820..bd79f146b 100644
--- a/plugin/gomark/parser/heading.go
+++ b/plugin/gomark/parser/heading.go
@@ -16,7 +16,7 @@ func NewHeadingParser() *HeadingParser {
func (*HeadingParser) Match(tokens []*tokenizer.Token) (int, bool) {
cursor := 0
for _, token := range tokens {
- if token.Type == tokenizer.Hash {
+ if token.Type == tokenizer.PoundSign {
cursor++
} else {
break
@@ -57,7 +57,7 @@ func (p *HeadingParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
level := 0
for _, token := range tokens {
- if token.Type == tokenizer.Hash {
+ if token.Type == tokenizer.PoundSign {
level++
} else {
break
diff --git a/plugin/gomark/parser/horizontal_rule.go b/plugin/gomark/parser/horizontal_rule.go
index a3db6d1d4..75d6c50a2 100644
--- a/plugin/gomark/parser/horizontal_rule.go
+++ b/plugin/gomark/parser/horizontal_rule.go
@@ -20,7 +20,7 @@ func (*HorizontalRuleParser) Match(tokens []*tokenizer.Token) (int, bool) {
if tokens[0].Type != tokens[1].Type || tokens[0].Type != tokens[2].Type || tokens[1].Type != tokens[2].Type {
return 0, false
}
- if tokens[0].Type != tokenizer.Dash && tokens[0].Type != tokenizer.Underline && tokens[0].Type != tokenizer.Asterisk {
+ if tokens[0].Type != tokenizer.Hyphen && tokens[0].Type != tokenizer.Underscore && tokens[0].Type != tokenizer.Asterisk {
return 0, false
}
if len(tokens) > 3 && tokens[3].Type != tokenizer.Newline {
diff --git a/plugin/gomark/parser/italic.go b/plugin/gomark/parser/italic.go
index 1b29afeb0..6a80496c1 100644
--- a/plugin/gomark/parser/italic.go
+++ b/plugin/gomark/parser/italic.go
@@ -21,7 +21,7 @@ func (*ItalicParser) Match(tokens []*tokenizer.Token) (int, bool) {
}
prefixTokens := tokens[:1]
- if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underline {
+ if prefixTokens[0].Type != tokenizer.Asterisk && prefixTokens[0].Type != tokenizer.Underscore {
return 0, false
}
prefixTokenType := prefixTokens[0].Type
diff --git a/plugin/gomark/parser/ordered_list.go b/plugin/gomark/parser/ordered_list.go
new file mode 100644
index 000000000..4e01b4760
--- /dev/null
+++ b/plugin/gomark/parser/ordered_list.go
@@ -0,0 +1,54 @@
+package parser
+
+import (
+ "errors"
+
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+type OrderedListParser struct{}
+
+func NewOrderedListParser() *OrderedListParser {
+ return &OrderedListParser{}
+}
+
+func (*OrderedListParser) Match(tokens []*tokenizer.Token) (int, bool) {
+ if len(tokens) < 4 {
+ return 0, false
+ }
+ if tokens[0].Type != tokenizer.Number || tokens[1].Type != tokenizer.Dot || tokens[2].Type != tokenizer.Space {
+ return 0, false
+ }
+
+ contentTokens := []*tokenizer.Token{}
+ for _, token := range tokens[3:] {
+ contentTokens = append(contentTokens, token)
+ if token.Type == tokenizer.Newline {
+ break
+ }
+ }
+
+ if len(contentTokens) == 0 {
+ return 0, false
+ }
+
+ return len(contentTokens) + 3, true
+}
+
+func (p *OrderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
+ size, ok := p.Match(tokens)
+ if size == 0 || !ok {
+ return nil, errors.New("not matched")
+ }
+
+ contentTokens := tokens[3:size]
+ children, err := ParseInline(contentTokens)
+ if err != nil {
+ return nil, err
+ }
+ return &ast.OrderedList{
+ Number: tokens[0].Value,
+ Children: children,
+ }, nil
+}
diff --git a/plugin/gomark/parser/ordered_list_test.go b/plugin/gomark/parser/ordered_list_test.go
new file mode 100644
index 000000000..374c317af
--- /dev/null
+++ b/plugin/gomark/parser/ordered_list_test.go
@@ -0,0 +1,58 @@
+package parser
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+func TestOrderedListParser(t *testing.T) {
+ tests := []struct {
+ text string
+ node ast.Node
+ }{
+ {
+ text: "1.asd",
+ node: nil,
+ },
+ {
+ text: "1. Hello World",
+ node: &ast.OrderedList{
+ Number: "1",
+ Children: []ast.Node{
+ &ast.Text{
+ Content: "Hello World",
+ },
+ },
+ },
+ },
+ {
+ text: "1aa. Hello World",
+ node: nil,
+ },
+ {
+ text: "22. Hello *World*",
+ node: &ast.OrderedList{
+ Number: "22",
+ Children: []ast.Node{
+ &ast.Text{
+ Content: "Hello ",
+ },
+ &ast.Italic{
+ Symbol: "*",
+ Content: "World",
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ tokens := tokenizer.Tokenize(test.text)
+ node, _ := NewOrderedListParser().Parse(tokens)
+ require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node}))
+ }
+}
diff --git a/plugin/gomark/parser/tag.go b/plugin/gomark/parser/tag.go
index 3deccad39..9a7d872f1 100644
--- a/plugin/gomark/parser/tag.go
+++ b/plugin/gomark/parser/tag.go
@@ -17,12 +17,12 @@ func (*TagParser) Match(tokens []*tokenizer.Token) (int, bool) {
if len(tokens) < 2 {
return 0, false
}
- if tokens[0].Type != tokenizer.Hash {
+ if tokens[0].Type != tokenizer.PoundSign {
return 0, false
}
contentTokens := []*tokenizer.Token{}
for _, token := range tokens[1:] {
- if token.Type == tokenizer.Newline || token.Type == tokenizer.Space || token.Type == tokenizer.Hash {
+ if token.Type == tokenizer.Newline || token.Type == tokenizer.Space || token.Type == tokenizer.PoundSign {
break
}
contentTokens = append(contentTokens, token)
diff --git a/plugin/gomark/parser/tokenizer/tokenizer.go b/plugin/gomark/parser/tokenizer/tokenizer.go
index ce3730014..e138ba829 100644
--- a/plugin/gomark/parser/tokenizer/tokenizer.go
+++ b/plugin/gomark/parser/tokenizer/tokenizer.go
@@ -3,9 +3,9 @@ package tokenizer
type TokenType = string
const (
- Underline TokenType = "_"
+ Underscore TokenType = "_"
Asterisk TokenType = "*"
- Hash TokenType = "#"
+ PoundSign TokenType = "#"
Backtick TokenType = "`"
LeftSquareBracket TokenType = "["
RightSquareBracket TokenType = "]"
@@ -13,14 +13,17 @@ const (
RightParenthesis TokenType = ")"
ExclamationMark TokenType = "!"
Tilde TokenType = "~"
- Dash TokenType = "-"
+ Hyphen TokenType = "-"
+ PlusSign TokenType = "+"
+ Dot TokenType = "."
GreaterThan TokenType = ">"
Newline TokenType = "\n"
Space TokenType = " "
)
const (
- Text TokenType = ""
+ Number TokenType = "number"
+ Text TokenType = ""
)
type Token struct {
@@ -40,11 +43,11 @@ func Tokenize(text string) []*Token {
for _, c := range text {
switch c {
case '_':
- tokens = append(tokens, NewToken(Underline, "_"))
+ tokens = append(tokens, NewToken(Underscore, "_"))
case '*':
tokens = append(tokens, NewToken(Asterisk, "*"))
case '#':
- tokens = append(tokens, NewToken(Hash, "#"))
+ tokens = append(tokens, NewToken(PoundSign, "#"))
case '`':
tokens = append(tokens, NewToken(Backtick, "`"))
case '[':
@@ -60,9 +63,13 @@ func Tokenize(text string) []*Token {
case '~':
tokens = append(tokens, NewToken(Tilde, "~"))
case '-':
- tokens = append(tokens, NewToken(Dash, "-"))
+ tokens = append(tokens, NewToken(Hyphen, "-"))
case '>':
tokens = append(tokens, NewToken(GreaterThan, ">"))
+ case '+':
+ tokens = append(tokens, NewToken(PlusSign, "+"))
+ case '.':
+ tokens = append(tokens, NewToken(Dot, "."))
case '\n':
tokens = append(tokens, NewToken(Newline, "\n"))
case ' ':
@@ -72,10 +79,19 @@ func Tokenize(text string) []*Token {
if len(tokens) > 0 {
prevToken = tokens[len(tokens)-1]
}
- if prevToken == nil || prevToken.Type != Text {
- tokens = append(tokens, NewToken(Text, string(c)))
+
+ isNumber := c >= '0' && c <= '9'
+ if prevToken != nil {
+ if (prevToken.Type == Text && !isNumber) || (prevToken.Type == Number && isNumber) {
+ prevToken.Value += string(c)
+ continue
+ }
+ }
+
+ if isNumber {
+ tokens = append(tokens, NewToken(Number, string(c)))
} else {
- prevToken.Value += string(c)
+ tokens = append(tokens, NewToken(Text, string(c)))
}
}
}
diff --git a/plugin/gomark/parser/tokenizer/tokenizer_test.go b/plugin/gomark/parser/tokenizer/tokenizer_test.go
index a85651b3d..284dba939 100644
--- a/plugin/gomark/parser/tokenizer/tokenizer_test.go
+++ b/plugin/gomark/parser/tokenizer/tokenizer_test.go
@@ -41,7 +41,7 @@ func TestTokenize(t *testing.T) {
world`,
tokens: []*Token{
{
- Type: Hash,
+ Type: PoundSign,
Value: "#",
},
{
diff --git a/plugin/gomark/parser/unordered_list.go b/plugin/gomark/parser/unordered_list.go
new file mode 100644
index 000000000..c64daeb17
--- /dev/null
+++ b/plugin/gomark/parser/unordered_list.go
@@ -0,0 +1,55 @@
+package parser
+
+import (
+ "errors"
+
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+type UnorderedListParser struct{}
+
+func NewUnorderedListParser() *UnorderedListParser {
+ return &UnorderedListParser{}
+}
+
+func (*UnorderedListParser) Match(tokens []*tokenizer.Token) (int, bool) {
+ if len(tokens) < 3 {
+ return 0, false
+ }
+ symbolToken := tokens[0]
+ if (symbolToken.Type != tokenizer.Hyphen && symbolToken.Type != tokenizer.Asterisk && symbolToken.Type != tokenizer.PlusSign) || tokens[1].Type != tokenizer.Space {
+ return 0, false
+ }
+
+ contentTokens := []*tokenizer.Token{}
+ for _, token := range tokens[2:] {
+ if token.Type == tokenizer.Newline {
+ break
+ }
+ contentTokens = append(contentTokens, token)
+ }
+ if len(contentTokens) == 0 {
+ return 0, false
+ }
+
+ return len(contentTokens) + 2, true
+}
+
+func (p *UnorderedListParser) Parse(tokens []*tokenizer.Token) (ast.Node, error) {
+ size, ok := p.Match(tokens)
+ if size == 0 || !ok {
+ return nil, errors.New("not matched")
+ }
+
+ symbolToken := tokens[0]
+ contentTokens := tokens[2:size]
+ children, err := ParseInline(contentTokens)
+ if err != nil {
+ return nil, err
+ }
+ return &ast.UnorderedList{
+ Symbol: symbolToken.Type,
+ Children: children,
+ }, nil
+}
diff --git a/plugin/gomark/parser/unordered_list_test.go b/plugin/gomark/parser/unordered_list_test.go
new file mode 100644
index 000000000..982d58be7
--- /dev/null
+++ b/plugin/gomark/parser/unordered_list_test.go
@@ -0,0 +1,51 @@
+package parser
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/usememos/memos/plugin/gomark/ast"
+ "github.com/usememos/memos/plugin/gomark/parser/tokenizer"
+)
+
+func TestUnorderedListParser(t *testing.T) {
+ tests := []struct {
+ text string
+ node ast.Node
+ }{
+ {
+ text: "*asd",
+ node: nil,
+ },
+ {
+ text: "+ Hello World",
+ node: &ast.UnorderedList{
+ Symbol: tokenizer.PlusSign,
+ Children: []ast.Node{
+ &ast.Text{
+ Content: "Hello World",
+ },
+ },
+ },
+ },
+ {
+ text: "* **Hello**",
+ node: &ast.UnorderedList{
+ Symbol: tokenizer.Asterisk,
+ Children: []ast.Node{
+ &ast.Bold{
+ Symbol: "*",
+ Content: "Hello",
+ },
+ },
+ },
+ },
+ }
+
+ for _, test := range tests {
+ tokens := tokenizer.Tokenize(test.text)
+ node, _ := NewUnorderedListParser().Parse(tokens)
+ require.Equal(t, StringifyNodes([]ast.Node{test.node}), StringifyNodes([]ast.Node{node}))
+ }
+}
diff --git a/plugin/gomark/render/html/html.go b/plugin/gomark/render/html/html.go
index aedbd222b..46f90bc66 100644
--- a/plugin/gomark/render/html/html.go
+++ b/plugin/gomark/render/html/html.go
@@ -159,7 +159,7 @@ func (r *HTMLRender) renderLink(node *ast.Link) {
func (r *HTMLRender) renderTag(node *ast.Tag) {
r.output.WriteString(`<span>`)
- r.output.WriteString(`# `)
+ r.output.WriteString(`#`)
r.output.WriteString(node.Content)
r.output.WriteString(`</span>`)
}
diff --git a/plugin/gomark/render/html/html_test.go b/plugin/gomark/render/html/html_test.go
index 9c22097ac..25164254d 100644
--- a/plugin/gomark/render/html/html_test.go
+++ b/plugin/gomark/render/html/html_test.go
@@ -30,6 +30,10 @@ func TestHTMLRender(t *testing.T) {
text: "**Hello** world!",
expected: `<p><strong>Hello</strong> world!</p>`,
},
+ {
+ text: "#article #memo",
+ expected: `<p><span>#article</span> <span>#memo</span></p>`,
+ },
}
for _, test := range tests {