Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

10.1 KB
Raw Blame
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later

package model

import (
	"bytes"
	"errors"
	"fmt"
	"os"
	"strings"
	"testing"

	"github.com/88250/lute/ast"
	"github.com/88250/lute/parse"
	"github.com/88250/lute/render"
	"github.com/siyuan-note/siyuan/kernel/conf"
	"github.com/siyuan-note/siyuan/kernel/treenode"
	"github.com/siyuan-note/siyuan/kernel/util"
)

func setupMarkdownSpoolTest(t *testing.T) {
	t.Helper()
	originalConf, originalTemp := Conf, util.TempDir
	Conf = NewAppConf()
	{
		confSectionValue0, confSectionValue1 := conf.NewEditor(), conf.NewExport()
		Conf.Editor.Store(confSectionValue0)
		Conf.Export.Store(confSectionValue1)
	}
	util.TempDir = t.TempDir()
	t.Cleanup(func() { Conf, util.TempDir = originalConf, originalTemp })
}

func markdownSpoolFixture(markdown, title string, document int) *parse.Tree {
	tree, _, _, _ := parseStdMd([]byte(markdown), true)
	tree.Root.ID = tree.Root.IALAttr("id")
	ids := map[string]string{}
	count := 0
	ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
		if entering && n.ID != "" {
			id := fmt.Sprintf("20261010000000-%07d", document*1000+count)
			count++
			ids[n.ID] = id
			n.ID = id
			n.SetIALAttr("id", id)
			n.SetIALAttr("updated", "20261010000000")
		}
		return ast.WalkContinue
	})
	ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
		if entering && treenode.IsBlockRef(n) {
			if id := ids[n.TextMarkBlockRefID]; id != "" {
				n.TextMarkBlockRefID = id
			}
		}
		return ast.WalkContinue
	})
	tree.ID, tree.Box = tree.Root.ID, "20261010000000-box0001"
	tree.Path, tree.HPath = "/"+tree.ID+".sy", "/"+title
	tree.Root.Spec = treenode.CurrentSpec
	tree.Root.SetIALAttr("title", title)
	return tree
}

func renderMarkdownSpoolFixture(tree *parse.Tree) []byte {
	engine := util.NewLute()
	return render.NewJSONRenderer(tree, engine.RenderOptions, engine.ParseOptions).Render()
}

func TestMarkdownImportSpoolPreservesConversion(t *testing.T) {
	setupMarkdownSpoolTest(t)
	spool, err := newMarkdownImportSpool()
	if err != nil {
		t.Fatal(err)
	}
	defer spool.close()
	fixtures := []struct{ title, markdown string }{
		{"one", "# Heading one\n\n[[two#Heading two|forward]] and [document](two.md).\n\n*em* **strong** `code` $x+y$ #tag\n\nText[^note].\n\n[^note]: Footnote **content**\n\n| a | b |\n| -- | -- |\n| 1 | 2 |\n\n```go\nfmt.Println(1)\n```\n\n<div>HTML <b>bold</b></div>\n"},
		{"two", "# Heading two\n\n[[one#Heading one]] and [[missing]].\n\n> Quote\n\n- [x] Task\n- Item\n\n![image](assets/image.png)\n"},
	}
	links := map[string]string{}
	var original []*parse.Tree
	for i, fixture := range fixtures {
		tree := markdownSpoolFixture(fixture.markdown, fixture.title, i+1)
		original = append(original, tree)
		addImportSearchLinks(tree, links)
		if err := spool.add(markdownSpoolFixture(fixture.markdown, fixture.title, i+1)); err != nil {
			t.Fatal(err)
		}
	}
	engine := NewLute()
	engine.SetHTMLTag2TextMark(true)
	var expected [][]byte
	for _, tree := range original {
		convertMdHyperlinks2WikiLinks(tree)
		convertWikiLinksAndTags(tree, links)
		mergeTextAndHandlerNestedInlines(tree, engine)
		expected = append(expected, renderMarkdownSpoolFixture(tree))
	}
	i := 0
	if err := spool.finish(func(tree *parse.Tree) error {
		if got := renderMarkdownSpoolFixture(tree); !bytes.Equal(got, expected[i]) {
			t.Errorf("staging changed document %d\ngot: %s\nwant: %s", i, got, expected[i])
		}
		i++
		return nil
	}, func() {}); err != nil {
		t.Fatal(err)
	}
	if i != len(fixtures) {
		t.Fatal("conversion omitted a document")
	}
}

func TestMarkdownImportHeadingFragments(t *testing.T) {
	setupMarkdownSpoolTest(t)
	target := markdownSpoolFixture("# Section\n\nTarget", "two", 2)
	headingID := target.Root.ChildrenByType(ast.NodeHeading)[0].ID
	links := map[string]string{}
	addImportSearchLinks(target, links)
	for _, extension := range []string{".md", ".markdown"} {
		t.Run(extension, func(t *testing.T) {
			tree := markdownSpoolFixture("[document](two"+extension+") [section](two"+extension+"#Section) [empty](two"+extension+"#) [asset](assets/two"+extension+"#Section) [web](https://example.com/page.html#Section)", "one", 1)
			if err := convertMarkdownImportTree(tree, links, NewLute()); err != nil {
				t.Fatal(err)
			}
			refs, hrefs := map[string]string{}, map[string]string{}
			ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
				if entering && treenode.IsBlockRef(n) {
					refs[n.TextMarkTextContent] = n.TextMarkBlockRefID
				}
				if entering && n.IsTextMarkType("a") {
					hrefs[n.TextMarkTextContent] = n.TextMarkAHref
				}
				return ast.WalkContinue
			})
			for label, want := range map[string]string{"document": target.ID, "section": headingID, "empty": target.ID} {
				if refs[label] != want {
					t.Errorf("%s target = %q, want %q", label, refs[label], want)
				}
			}
			if hrefs["asset"] != "assets/two"+extension+"#Section" || hrefs["web"] != "https://example.com/page.html#Section" {
				t.Fatalf("unrelated links changed: %v", hrefs)
			}
		})
	}
}

func TestMarkdownImportSpoolSeparatesFilePathsFromTitles(t *testing.T) {
	setupMarkdownSpoolTest(t)
	spool, err := newMarkdownImportSpool()
	if err != nil {
		t.Fatal(err)
	}
	defer spool.close()
	parent := markdownSpoolFixture("[[folder/child|child]]", "Parent/../title", 1)
	child := markdownSpoolFixture("[[../folder|parent]]", "Child//title", 2)
	child.Path = "/" + parent.ID + "/" + child.ID + ".sy"
	for _, fixture := range []struct {
		tree   *parse.Tree
		source string
	}{
		{parent, "/folder"}, {child, "/folder/child"},
	} {
		if err = spool.addSource(fixture.tree, fixture.source); err != nil {
			t.Fatal(err)
		}
	}
	if err = spool.refreshDisplayPaths("/", "/"); err != nil {
		t.Fatal(err)
	}
	expected := map[string]string{parent.ID: child.ID, child.ID: parent.ID}
	if err = spool.finish(func(tree *parse.Tree) error {
		got := ""
		ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
			if entering && treenode.IsBlockRef(n) {
				got = n.TextMarkBlockRefID
			}
			return ast.WalkContinue
		})
		if got != expected[tree.ID] {
			t.Fatalf("file link resolved through title: %s => %s", tree.ID, got)
		}
		wantPath := "/Parent/../title"
		if tree.ID == child.ID {
			wantPath += "/Child//title"
		}
		if tree.HPath != wantPath {
			t.Fatalf("display path: %q", tree.HPath)
		}
		return nil
	}, func() {}); err != nil {
		t.Fatal(err)
	}
}

func TestMarkdownImportSpoolBoundsQueuedTrees(t *testing.T) {
	setupMarkdownSpoolTest(t)
	spool, err := newMarkdownImportSpool()
	if err != nil {
		t.Fatal(err)
	}
	defer spool.close()
	for i := range 2*markdownImportBatchDocuments + 1 {
		if err := spool.add(markdownSpoolFixture("# Heading\n\ncontent", fmt.Sprint(i), i)); err != nil {
			t.Fatal(err)
		}
	}
	var queued []*parse.Tree
	flushed := 0
	if err := spool.finish(func(tree *parse.Tree) error {
		queued = append(queued, tree)
		if len(queued) > markdownImportBatchDocuments {
			t.Fatal("queue retained more than one batch of trees")
		}
		return nil
	}, func() {
		flushed += len(queued)
		queued = nil
	}); err != nil {
		t.Fatal(err)
	}
	if flushed != len(spool.entries) || len(queued) != 0 {
		t.Fatal("final indexing batch was not drained")
	}
}

func TestMarkdownImportSpoolAuthenticatesAndCleansUp(t *testing.T) {
	setupMarkdownSpoolTest(t)
	spool, err := newMarkdownImportSpool()
	if err != nil {
		t.Fatal(err)
	}
	defer spool.close()
	const secret = "private Markdown content must not be staged in plaintext"
	if err := spool.add(markdownSpoolFixture(secret, "private", 1)); err != nil {
		t.Fatal(err)
	}
	fileName := spool.file.Name()
	ciphertext, err := os.ReadFile(fileName)
	if err != nil {
		t.Fatal(err)
	}
	if bytes.Contains(ciphertext, []byte(secret)) {
		t.Fatal("staging file exposed plaintext")
	}
	if _, err := util.DecryptWithAAD(make([]byte, 32), ciphertext, markdownImportAAD(0)); err == nil {
		t.Fatal("wrong key authenticated staged data")
	}
	if _, err := util.DecryptWithAAD(spool.key, ciphertext, markdownImportAAD(1)); err == nil {
		t.Fatal("staged data was not bound to its entry")
	}
	ciphertext[len(ciphertext)-1] ^= 1
	if _, err := spool.file.WriteAt(ciphertext, 0); err != nil {
		t.Fatal(err)
	}
	if err := spool.finish(func(*parse.Tree) error {
		t.Fatal("unauthenticated staged data reached the writer")
		return nil
	}, func() {}); err == nil {
		t.Fatal("corrupted staging entry was accepted")
	}
	if got, err := os.ReadFile(fileName); err != nil || !bytes.Equal(got, ciphertext) {
		t.Fatal("authentication failure rewrote staged ciphertext")
	}
	key := spool.key
	spool.close()
	if _, err := os.Stat(fileName); !os.IsNotExist(err) {
		t.Fatal("temporary staging file was not removed")
	}
	if !bytes.Equal(key, make([]byte, len(key))) {
		t.Fatal("staging key was not cleared")
	}
}

func TestMarkdownImportSpoolFlushesOnWriteFailure(t *testing.T) {
	setupMarkdownSpoolTest(t)
	spool, err := newMarkdownImportSpool()
	if err != nil {
		t.Fatal(err)
	}
	defer spool.close()
	for i := range 3 {
		if err := spool.add(markdownSpoolFixture("content", fmt.Sprint(i), i)); err != nil {
			t.Fatal(err)
		}
	}
	writes, flushes := 0, 0
	failure := errors.New("disk write failed")
	err = spool.finish(func(*parse.Tree) error {
		writes++
		if writes == 2 {
			return failure
		}
		return nil
	}, func() { flushes++ })
	if !errors.Is(err, failure) || writes != 2 || flushes != 1 {
		t.Fatalf("write error or pending batch lost: %v, writes=%d, flushes=%d", err, writes, flushes)
	}
}

func TestMarkdownImportSpoolFlushesByBytes(t *testing.T) {
	setupMarkdownSpoolTest(t)
	spool, err := newMarkdownImportSpool()
	if err != nil {
		t.Fatal(err)
	}
	defer spool.close()
	if err := spool.add(markdownSpoolFixture(strings.Repeat("x", markdownImportBatchBytes), "large", 1)); err != nil {
		t.Fatal(err)
	}
	if err := spool.add(markdownSpoolFixture("small", "small", 2)); err != nil {
		t.Fatal(err)
	}
	writes, firstFlush := 0, 0
	if err := spool.finish(func(*parse.Tree) error {
		writes++
		return nil
	}, func() {
		if firstFlush == 0 {
			firstFlush = writes
		}
	}); err != nil {
		t.Fatal(err)
	}
	if firstFlush != 1 {
		t.Fatal("large document did not drain the queue by its byte budget")
	}
}