Skip to content

armgit / memos

Public Imported from GitHub
0

Memos is an open-source, self-hosted note-taking app that brings your everyday thoughts together in a personal timeline. Your data stays yours, and you choose what to keep private or share.

8.2 KB
Raw Blame
// Command tagdata generates the fixed Unicode and emoji tables used by the
// Memos tag lexer.
package main

import (
	"bufio"
	"bytes"
	"cmp"
	"crypto/sha256"
	"flag"
	"fmt"
	"go/format"
	"io"
	"log"
	"net/http"
	"os"
	"slices"
	"strconv"
	"strings"
	"time"

	"github.com/pkg/errors"
)

const (
	unicodeRelease           = "17.0.0"
	derivedCorePropertiesURL = "https://www.unicode.org/Public/" + unicodeRelease + "/ucd/DerivedCoreProperties.txt"
	unicodeDataURL           = "https://www.unicode.org/Public/" + unicodeRelease + "/ucd/UnicodeData.txt"
	emojiTestURL             = "https://www.unicode.org/Public/" + unicodeRelease + "/emoji/emoji-test.txt"
)

type codePointRange struct {
	lo rune
	hi rune
}

func main() {
	output := flag.String("output", "tag_unicode_tables.go", "generated Go output path")
	flag.Parse()

	client := &http.Client{Timeout: 30 * time.Second}
	derivedCoreProperties, err := fetch(client, derivedCorePropertiesURL)
	if err != nil {
		log.Fatal(err)
	}
	unicodeData, err := fetch(client, unicodeDataURL)
	if err != nil {
		log.Fatal(err)
	}
	emojiTest, err := fetch(client, emojiTestURL)
	if err != nil {
		log.Fatal(err)
	}

	xidContinue := parseDerivedProperty(derivedCoreProperties, "XID_Continue")
	defaultIgnorable := parseDerivedProperty(derivedCoreProperties, "Default_Ignorable_Code_Point")
	combiningMarks := parseCombiningMarks(unicodeData)
	emojis, maxEmojiBytes := parseFullyQualifiedEmoji(emojiTest)

	var generated bytes.Buffer
	generated.WriteString("// Code generated by go generate; DO NOT EDIT.\n")
	fmt.Fprintf(&generated, "// Unicode and Emoji %s data for Memos tag recognition.\n", unicodeRelease)
	fmt.Fprintf(&generated, "// Unicode data SHA-256: %s\n\n", unicodeDataDigest(xidContinue, defaultIgnorable, combiningMarks, emojis))
	generated.WriteString("// Sources:\n")
	fmt.Fprintf(&generated, "//   - %s\n", derivedCorePropertiesURL)
	fmt.Fprintf(&generated, "//   - %s\n", unicodeDataURL)
	fmt.Fprintf(&generated, "//   - %s\n\n", emojiTestURL)
	generated.WriteString("package parser\n\n")
	writeRanges(&generated, "unicode17XIDContinue", xidContinue)
	writeRanges(&generated, "unicode17DefaultIgnorable", defaultIgnorable)
	writeRanges(&generated, "unicode17CombiningMarks", combiningMarks)
	fmt.Fprintf(&generated, "const maxEmoji17SequenceBytes = %d\n\n", maxEmojiBytes)
	generated.WriteString("var emoji17FullyQualified = map[string]struct{}{\n")
	for _, emoji := range emojis {
		fmt.Fprintf(&generated, "\t%s: {},\n", strconv.QuoteToASCII(emoji))
	}
	generated.WriteString("}\n")

	formatted, err := format.Source(generated.Bytes())
	if err != nil {
		log.Fatalf("format generated source: %v", err)
	}
	if err := os.WriteFile(*output, formatted, 0o644); err != nil {
		log.Fatalf("write %s: %v", *output, err)
	}
}

func fetch(client *http.Client, url string) (data []byte, err error) {
	response, err := client.Get(url)
	if err != nil {
		return nil, errors.Wrapf(err, "fetch %s", url)
	}
	defer func() {
		if closeErr := response.Body.Close(); err == nil && closeErr != nil {
			err = errors.Wrapf(closeErr, "close %s", url)
		}
	}()
	if response.StatusCode != http.StatusOK {
		return nil, errors.Errorf("fetch %s: %s", url, response.Status)
	}
	data, err = io.ReadAll(response.Body)
	if err != nil {
		return nil, errors.Wrapf(err, "read %s", url)
	}
	return data, nil
}

func parseDerivedProperty(data []byte, property string) []codePointRange {
	var ranges []codePointRange
	scanner := bufio.NewScanner(bytes.NewReader(data))
	for scanner.Scan() {
		line := strings.TrimSpace(strings.SplitN(scanner.Text(), "#", 2)[0])
		if line == "" {
			continue
		}
		fields := strings.Split(line, ";")
		if len(fields) != 2 || strings.TrimSpace(fields[1]) != property {
			continue
		}
		ranges = append(ranges, parseRange(strings.TrimSpace(fields[0])))
	}
	if err := scanner.Err(); err != nil {
		log.Fatalf("scan DerivedCoreProperties.txt: %v", err) //nolint:revive // Malformed generator input is fatal.
	}
	return mergeRanges(ranges)
}

func parseCombiningMarks(data []byte) []codePointRange {
	var ranges []codePointRange
	var rangeStart rune = -1
	scanner := bufio.NewScanner(bytes.NewReader(data))
	for scanner.Scan() {
		if strings.TrimSpace(scanner.Text()) == "" {
			continue
		}
		fields := strings.Split(scanner.Text(), ";")
		if len(fields) < 3 {
			log.Fatalf("invalid UnicodeData.txt line: %q", scanner.Text()) //nolint:revive // Malformed generator input is fatal.
		}
		codePoint := parseCodePoint(fields[0])
		name := fields[1]
		category := fields[2]
		isCombining := category == "Mn" || category == "Mc"
		if strings.HasSuffix(name, ", First>") {
			if isCombining {
				rangeStart = codePoint
			}
			continue
		}
		if strings.HasSuffix(name, ", Last>") {
			if rangeStart >= 0 {
				ranges = append(ranges, codePointRange{lo: rangeStart, hi: codePoint})
				rangeStart = -1
			}
			continue
		}
		if isCombining {
			ranges = append(ranges, codePointRange{lo: codePoint, hi: codePoint})
		}
	}
	if err := scanner.Err(); err != nil {
		log.Fatalf("scan UnicodeData.txt: %v", err) //nolint:revive // Malformed generator input is fatal.
	}
	return mergeRanges(ranges)
}

func unicodeDataDigest(xidContinue, defaultIgnorable, combiningMarks []codePointRange, emojis []string) string {
	var canonical strings.Builder
	writeRanges := func(name string, ranges []codePointRange) {
		for _, codeRange := range ranges {
			fmt.Fprintf(&canonical, "%s:%x-%x\n", name, codeRange.lo, codeRange.hi)
		}
	}
	writeRanges("xid", xidContinue)
	writeRanges("ignorable", defaultIgnorable)
	writeRanges("combining", combiningMarks)

	emojiLines := make([]string, 0, len(emojis))
	for _, emoji := range emojis {
		codePoints := make([]string, 0, len(emoji))
		for _, codePoint := range emoji {
			codePoints = append(codePoints, strconv.FormatInt(int64(codePoint), 16))
		}
		emojiLines = append(emojiLines, "emoji:"+strings.Join(codePoints, ",")+"\n")
	}
	slices.Sort(emojiLines)
	for _, line := range emojiLines {
		canonical.WriteString(line)
	}

	digest := sha256.Sum256([]byte(canonical.String()))
	return fmt.Sprintf("%x", digest)
}

func parseFullyQualifiedEmoji(data []byte) ([]string, int) {
	var emojis []string
	maxBytes := 0
	scanner := bufio.NewScanner(bytes.NewReader(data))
	for scanner.Scan() {
		line := strings.TrimSpace(strings.SplitN(scanner.Text(), "#", 2)[0])
		if line == "" {
			continue
		}
		fields := strings.Split(line, ";")
		if len(fields) != 2 || strings.TrimSpace(fields[1]) != "fully-qualified" {
			continue
		}
		var sequence strings.Builder
		for field := range strings.FieldsSeq(fields[0]) {
			sequence.WriteRune(parseCodePoint(field))
		}
		emoji := sequence.String()
		emojis = append(emojis, emoji)
		maxBytes = max(maxBytes, len(emoji))
	}
	if err := scanner.Err(); err != nil {
		log.Fatalf("scan emoji-test.txt: %v", err) //nolint:revive // Malformed generator input is fatal.
	}
	slices.Sort(emojis)
	return emojis, maxBytes
}

func parseRange(value string) codePointRange {
	parts := strings.Split(value, "..")
	if len(parts) == 1 {
		codePoint := parseCodePoint(parts[0])
		return codePointRange{lo: codePoint, hi: codePoint}
	}
	if len(parts) != 2 {
		log.Fatalf("invalid code-point range %q", value) //nolint:revive // Malformed generator input is fatal.
	}
	return codePointRange{lo: parseCodePoint(parts[0]), hi: parseCodePoint(parts[1])}
}

func parseCodePoint(value string) rune {
	codePoint, err := strconv.ParseInt(strings.TrimSpace(value), 16, 32)
	if err != nil {
		log.Fatalf("parse code point %q: %v", value, err) //nolint:revive // Malformed generator input is fatal.
	}
	return rune(codePoint)
}

func mergeRanges(ranges []codePointRange) []codePointRange {
	if len(ranges) == 0 {
		return nil
	}
	slices.SortFunc(ranges, func(a, b codePointRange) int { return cmp.Compare(a.lo, b.lo) })
	merged := []codePointRange{ranges[0]}
	for _, current := range ranges[1:] {
		last := &merged[len(merged)-1]
		if current.lo <= last.hi+1 {
			last.hi = max(last.hi, current.hi)
			continue
		}
		merged = append(merged, current)
	}
	return merged
}

func writeRanges(output *bytes.Buffer, name string, ranges []codePointRange) {
	fmt.Fprintf(output, "var %s = [...]codePointRange{\n", name)
	for _, codeRange := range ranges {
		fmt.Fprintf(output, "\t{lo: 0x%04X, hi: 0x%04X},\n", codeRange.lo, codeRange.hi)
	}
	output.WriteString("}\n\n")
}