Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.
Go dockerfossgomarkdownmemomicroblognote-takingnotesopen-sourceown-your-datareactself-hostedsqlitetimeline
// Command tagdata generates the fixed Unicode and emoji tables used by the
// Memos tag lexer.
package main
import (
"bufio"
"bytes"
"cmp"
"crypto/sha256"
"flag"
"fmt"
"go/format"
"io"
"log"
"net/http"
"os"
"slices"
"strconv"
"strings"
"time"
"github.com/pkg/errors"
)
const (
unicodeRelease = "17.0.0"
derivedCorePropertiesURL = "https://www.unicode.org/Public/" + unicodeRelease + "/ucd/DerivedCoreProperties.txt"
unicodeDataURL = "https://www.unicode.org/Public/" + unicodeRelease + "/ucd/UnicodeData.txt"
emojiTestURL = "https://www.unicode.org/Public/" + unicodeRelease + "/emoji/emoji-test.txt"
)
type codePointRange struct {
lo rune
hi rune
}
func main() {
output := flag.String("output", "tag_unicode_tables.go", "generated Go output path")
flag.Parse()
client := &http.Client{Timeout: 30 * time.Second}
derivedCoreProperties, err := fetch(client, derivedCorePropertiesURL)
if err != nil {
log.Fatal(err)
}
unicodeData, err := fetch(client, unicodeDataURL)
if err != nil {
log.Fatal(err)
}
emojiTest, err := fetch(client, emojiTestURL)
if err != nil {
log.Fatal(err)
}
xidContinue := parseDerivedProperty(derivedCoreProperties, "XID_Continue")
defaultIgnorable := parseDerivedProperty(derivedCoreProperties, "Default_Ignorable_Code_Point")
combiningMarks := parseCombiningMarks(unicodeData)
emojis, maxEmojiBytes := parseFullyQualifiedEmoji(emojiTest)
var generated bytes.Buffer
generated.WriteString("// Code generated by go generate; DO NOT EDIT.\n")
fmt.Fprintf(&generated, "// Unicode and Emoji %s data for Memos tag recognition.\n", unicodeRelease)
fmt.Fprintf(&generated, "// Unicode data SHA-256: %s\n\n", unicodeDataDigest(xidContinue, defaultIgnorable, combiningMarks, emojis))
generated.WriteString("// Sources:\n")
fmt.Fprintf(&generated, "// - %s\n", derivedCorePropertiesURL)
fmt.Fprintf(&generated, "// - %s\n", unicodeDataURL)
fmt.Fprintf(&generated, "// - %s\n\n", emojiTestURL)
generated.WriteString("package parser\n\n")
writeRanges(&generated, "unicode17XIDContinue", xidContinue)
writeRanges(&generated, "unicode17DefaultIgnorable", defaultIgnorable)
writeRanges(&generated, "unicode17CombiningMarks", combiningMarks)
fmt.Fprintf(&generated, "const maxEmoji17SequenceBytes = %d\n\n", maxEmojiBytes)
generated.WriteString("var emoji17FullyQualified = map[string]struct{}{\n")
for _, emoji := range emojis {
fmt.Fprintf(&generated, "\t%s: {},\n", strconv.QuoteToASCII(emoji))
}
generated.WriteString("}\n")
formatted, err := format.Source(generated.Bytes())
if err != nil {
log.Fatalf("format generated source: %v", err)
}
if err := os.WriteFile(*output, formatted, 0o644); err != nil {
log.Fatalf("write %s: %v", *output, err)
}
}
func fetch(client *http.Client, url string) (data []byte, err error) {
response, err := client.Get(url)
if err != nil {
return nil, errors.Wrapf(err, "fetch %s", url)
}
defer func() {
if closeErr := response.Body.Close(); err == nil && closeErr != nil {
err = errors.Wrapf(closeErr, "close %s", url)
}
}()
if response.StatusCode != http.StatusOK {
return nil, errors.Errorf("fetch %s: %s", url, response.Status)
}
data, err = io.ReadAll(response.Body)
if err != nil {
return nil, errors.Wrapf(err, "read %s", url)
}
return data, nil
}
func parseDerivedProperty(data []byte, property string) []codePointRange {
var ranges []codePointRange
scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
line := strings.TrimSpace(strings.SplitN(scanner.Text(), "#", 2)[0])
if line == "" {
continue
}
fields := strings.Split(line, ";")
if len(fields) != 2 || strings.TrimSpace(fields[1]) != property {
continue
}
ranges = append(ranges, parseRange(strings.TrimSpace(fields[0])))
}
if err := scanner.Err(); err != nil {
log.Fatalf("scan DerivedCoreProperties.txt: %v", err) //nolint:revive // Malformed generator input is fatal.
}
return mergeRanges(ranges)
}
func parseCombiningMarks(data []byte) []codePointRange {
var ranges []codePointRange
var rangeStart rune = -1
scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
if strings.TrimSpace(scanner.Text()) == "" {
continue
}
fields := strings.Split(scanner.Text(), ";")
if len(fields) < 3 {
log.Fatalf("invalid UnicodeData.txt line: %q", scanner.Text()) //nolint:revive // Malformed generator input is fatal.
}
codePoint := parseCodePoint(fields[0])
name := fields[1]
category := fields[2]
isCombining := category == "Mn" || category == "Mc"
if strings.HasSuffix(name, ", First>") {
if isCombining {
rangeStart = codePoint
}
continue
}
if strings.HasSuffix(name, ", Last>") {
if rangeStart >= 0 {
ranges = append(ranges, codePointRange{lo: rangeStart, hi: codePoint})
rangeStart = -1
}
continue
}
if isCombining {
ranges = append(ranges, codePointRange{lo: codePoint, hi: codePoint})
}
}
if err := scanner.Err(); err != nil {
log.Fatalf("scan UnicodeData.txt: %v", err) //nolint:revive // Malformed generator input is fatal.
}
return mergeRanges(ranges)
}
func unicodeDataDigest(xidContinue, defaultIgnorable, combiningMarks []codePointRange, emojis []string) string {
var canonical strings.Builder
writeRanges := func(name string, ranges []codePointRange) {
for _, codeRange := range ranges {
fmt.Fprintf(&canonical, "%s:%x-%x\n", name, codeRange.lo, codeRange.hi)
}
}
writeRanges("xid", xidContinue)
writeRanges("ignorable", defaultIgnorable)
writeRanges("combining", combiningMarks)
emojiLines := make([]string, 0, len(emojis))
for _, emoji := range emojis {
codePoints := make([]string, 0, len(emoji))
for _, codePoint := range emoji {
codePoints = append(codePoints, strconv.FormatInt(int64(codePoint), 16))
}
emojiLines = append(emojiLines, "emoji:"+strings.Join(codePoints, ",")+"\n")
}
slices.Sort(emojiLines)
for _, line := range emojiLines {
canonical.WriteString(line)
}
digest := sha256.Sum256([]byte(canonical.String()))
return fmt.Sprintf("%x", digest)
}
func parseFullyQualifiedEmoji(data []byte) ([]string, int) {
var emojis []string
maxBytes := 0
scanner := bufio.NewScanner(bytes.NewReader(data))
for scanner.Scan() {
line := strings.TrimSpace(strings.SplitN(scanner.Text(), "#", 2)[0])
if line == "" {
continue
}
fields := strings.Split(line, ";")
if len(fields) != 2 || strings.TrimSpace(fields[1]) != "fully-qualified" {
continue
}
var sequence strings.Builder
for field := range strings.FieldsSeq(fields[0]) {
sequence.WriteRune(parseCodePoint(field))
}
emoji := sequence.String()
emojis = append(emojis, emoji)
maxBytes = max(maxBytes, len(emoji))
}
if err := scanner.Err(); err != nil {
log.Fatalf("scan emoji-test.txt: %v", err) //nolint:revive // Malformed generator input is fatal.
}
slices.Sort(emojis)
return emojis, maxBytes
}
func parseRange(value string) codePointRange {
parts := strings.Split(value, "..")
if len(parts) == 1 {
codePoint := parseCodePoint(parts[0])
return codePointRange{lo: codePoint, hi: codePoint}
}
if len(parts) != 2 {
log.Fatalf("invalid code-point range %q", value) //nolint:revive // Malformed generator input is fatal.
}
return codePointRange{lo: parseCodePoint(parts[0]), hi: parseCodePoint(parts[1])}
}
func parseCodePoint(value string) rune {
codePoint, err := strconv.ParseInt(strings.TrimSpace(value), 16, 32)
if err != nil {
log.Fatalf("parse code point %q: %v", value, err) //nolint:revive // Malformed generator input is fatal.
}
return rune(codePoint)
}
func mergeRanges(ranges []codePointRange) []codePointRange {
if len(ranges) == 0 {
return nil
}
slices.SortFunc(ranges, func(a, b codePointRange) int { return cmp.Compare(a.lo, b.lo) })
merged := []codePointRange{ranges[0]}
for _, current := range ranges[1:] {
last := &merged[len(merged)-1]
if current.lo <= last.hi+1 {
last.hi = max(last.hi, current.hi)
continue
}
merged = append(merged, current)
}
return merged
}
func writeRanges(output *bytes.Buffer, name string, ranges []codePointRange) {
fmt.Fprintf(output, "var %s = [...]codePointRange{\n", name)
for _, codeRange := range ranges {
fmt.Fprintf(output, "\t{lo: 0x%04X, hi: 0x%04X},\n", codeRange.lo, codeRange.hi)
}
output.WriteString("}\n\n")
}