An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later
//go:build fts5
package model
import (
"os"
"os/exec"
"path"
"path/filepath"
"strings"
"testing"
"time"
"github.com/88250/lute/parse"
"github.com/siyuan-note/siyuan/kernel/cache"
"github.com/siyuan-note/siyuan/kernel/conf"
"github.com/siyuan-note/siyuan/kernel/filesys"
"github.com/siyuan-note/siyuan/kernel/sql"
"github.com/siyuan-note/siyuan/kernel/treenode"
"github.com/siyuan-note/siyuan/kernel/util"
)
const removeDocSQLTestEnv = "SIYUAN_TEST_REMOVE_DOC_SQL"
const docTemplateSQLTestEnv = "SIYUAN_TEST_DOC_TEMPLATE_SQL"
func TestRemoveDocFlushesDatabaseIndex(t *testing.T) {
if "1" != os.Getenv(removeDocSQLTestEnv) {
cmd := exec.Command(os.Args[0], "-test.run=^TestRemoveDocFlushesDatabaseIndex$", "-test.v")
cmd.Env = append(os.Environ(), removeDocSQLTestEnv+"=1")
output, err := cmd.CombinedOutput()
if nil != err {
t.Fatalf("remove document SQL subprocess failed: %v\n%s", err, output)
}
return
}
workspaceDir := t.TempDir()
util.WorkspaceDir = workspaceDir
util.ConfDir = filepath.Join(workspaceDir, "conf")
util.DataDir = filepath.Join(workspaceDir, "data")
util.HistoryDir = filepath.Join(workspaceDir, "history")
util.TempDir = filepath.Join(workspaceDir, "temp")
util.QueueDir = filepath.Join(util.TempDir, "queue")
util.DBPath = filepath.Join(util.TempDir, util.DBName)
util.HistoryDBPath = filepath.Join(util.TempDir, "history.db")
util.AssetContentDBPath = filepath.Join(util.TempDir, "asset_content.db")
util.BlockTreeDBPath = filepath.Join(util.TempDir, "blocktree.db")
for _, dir := range []string{util.ConfDir, util.DataDir, util.HistoryDir, util.TempDir, util.QueueDir} {
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatalf("create test directory [%s] failed: %v", dir, err)
}
}
Conf = NewAppConf()
Conf.FileTree.Store(conf.NewFileTree())
Conf.NotebookCrypto = conf.NewNotebookCrypto()
Conf.Sync.Store(conf.NewSync())
box := &Box{ID: "20260726000000-abcdefg"}
boxConf := conf.NewBoxConf()
boxConf.Name = "Remove document SQL test"
boxConf.Closed = false
if err := box.SaveConf(boxConf); err != nil {
t.Fatalf("save test notebook conf failed: %v", err)
}
sql.InitDatabase(true)
sql.InitHistoryDatabase(true)
sql.InitAssetContentDatabase(true)
t.Cleanup(sql.CloseDatabase)
tree := treenode.NewTree(box.ID, "/20260726000001-abcdefg.sy", "/Delete target", "Delete target")
if err := indexWriteTreeIndexQueue(tree); err != nil {
t.Fatalf("write and index test document failed: %v", err)
}
childTree := treenode.NewTree(
box.ID,
path.Join("/20260726000001-abcdefg", "20260726000002-abcdefg.sy"),
"/Delete target/Child",
"Child",
)
if err := indexWriteTreeIndexQueue(childTree); err != nil {
t.Fatalf("write and index child test document failed: %v", err)
}
sql.FlushQueue()
assertDatabaseBlockExists(t, tree.ID, true)
assertDatabaseBlockExists(t, childTree.ID, true)
blockIDs := []string{tree.ID, tree.Root.FirstChild.ID, childTree.ID, childTree.Root.FirstChild.ID}
for _, blockID := range blockIDs {
warmBlockIALCache(t, blockID, box.ID)
}
if _, ok := cache.GetTreeDataInBox(childTree.ID, box.ID); !ok {
t.Fatalf("child tree data cache [%s] was not populated", childTree.ID)
}
if ial := cache.GetDocIALInBox(childTree.Path, box.ID); nil == ial {
t.Fatalf("child document IAL cache [%s] was not populated", childTree.Path)
}
if err := RemoveDoc(box.ID, tree.Path); err != nil {
t.Fatalf("remove test document failed: %v", err)
}
for _, blockID := range blockIDs {
if ial := cache.GetBlockIALInBox(blockID, box.ID); nil != ial {
t.Fatalf("deleted block IAL cache [%s] was retained: %+v", blockID, ial)
}
if ial := sql.GetBlockAttrs(blockID); 0 < len(ial) {
t.Fatalf("deleted block attributes [%s] were retained: %+v", blockID, ial)
}
}
if attrs := sql.BatchGetBlockAttrs(blockIDs); 0 < len(attrs) {
t.Fatalf("deleted block attributes were retained in batch result: %+v", attrs)
}
for _, removedTree := range []*parse.Tree{tree, childTree} {
if _, ok := cache.GetTreeDataInBox(removedTree.ID, box.ID); ok {
t.Fatalf("deleted tree data cache [%s] was retained", removedTree.ID)
}
if ial := cache.GetDocIALInBox(removedTree.Path, box.ID); nil != ial {
t.Fatalf("deleted document IAL cache [%s] was retained: %+v", removedTree.Path, ial)
}
if _, err := filesys.LoadTree(box.ID, removedTree.Path, util.NewLute()); nil == err {
t.Fatalf("deleted tree [%s] was loaded from cache", removedTree.ID)
}
}
sql.FlushQueue()
assertDatabaseBlockExists(t, tree.ID, false)
assertDatabaseBlockExists(t, childTree.ID, false)
}
func warmBlockIALCache(t *testing.T, blockID, boxID string) {
t.Helper()
deadline := time.Now().Add(time.Second)
for {
if attrs := sql.GetBlockAttrs(blockID); 0 == len(attrs) {
t.Fatalf("block attributes [%s] were not found", blockID)
}
if nil != cache.GetBlockIALInBox(blockID, boxID) {
return
}
if deadline.Before(time.Now()) {
t.Fatalf("block IAL cache [%s] was not populated", blockID)
}
time.Sleep(time.Millisecond)
}
}
func TestDocumentTemplatesWaitForDatabaseIndex(t *testing.T) {
if "1" != os.Getenv(docTemplateSQLTestEnv) {
cmd := exec.Command(os.Args[0], "-test.run=^TestDocumentTemplatesWaitForDatabaseIndex$", "-test.v")
cmd.Env = append(os.Environ(), docTemplateSQLTestEnv+"=1")
output, err := cmd.CombinedOutput()
if nil != err {
t.Fatalf("document template SQL subprocess failed: %v\n%s", err, output)
}
return
}
workspaceDir := t.TempDir()
util.WorkspaceDir = workspaceDir
util.ConfDir = filepath.Join(workspaceDir, "conf")
util.DataDir = filepath.Join(workspaceDir, "data")
util.HistoryDir = filepath.Join(workspaceDir, "history")
util.TempDir = filepath.Join(workspaceDir, "temp")
util.QueueDir = filepath.Join(util.TempDir, "queue")
util.DBPath = filepath.Join(util.TempDir, util.DBName)
util.HistoryDBPath = filepath.Join(util.TempDir, "history.db")
util.AssetContentDBPath = filepath.Join(util.TempDir, "asset_content.db")
util.BlockTreeDBPath = filepath.Join(util.TempDir, "blocktree.db")
for _, dir := range []string{util.ConfDir, util.DataDir, util.HistoryDir, util.TempDir, util.QueueDir} {
if err := os.MkdirAll(dir, 0755); err != nil {
t.Fatalf("create test directory [%s] failed: %v", dir, err)
}
}
Conf = NewAppConf()
Conf.FileTree.Store(conf.NewFileTree())
Conf.Editor.Store(conf.NewEditor())
Conf.Export.Store(conf.NewExport())
Conf.NotebookCrypto = conf.NewNotebookCrypto()
Conf.Sync.Store(conf.NewSync())
Conf.Lang = "en"
util.WorkingDir = filepath.Clean(filepath.Join("..", "..", "app"))
initLang()
box := &Box{ID: "20260728000000-abcdefg"}
boxConf := conf.NewBoxConf()
boxConf.Name = "Document template SQL test"
boxConf.Closed = false
boxConf.DailyNoteSavePath = "/Daily note"
boxConf.DailyNoteTemplatePath = "/indexed.md"
if err := box.SaveConf(boxConf); err != nil {
t.Fatalf("save test notebook conf failed: %v", err)
}
templateDir := filepath.Join(util.DataDir, "templates")
if err := os.MkdirAll(templateDir, 0755); err != nil {
t.Fatalf("create templates directory failed: %v", err)
}
template := `.action{range queryBlocks "SELECT * FROM blocks WHERE id = '?' LIMIT 1" .id}.action{.ID}.action{end}`
if err := os.WriteFile(filepath.Join(templateDir, "indexed.md"), []byte(template), 0644); err != nil {
t.Fatalf("write test template failed: %v", err)
}
sql.InitDatabase(true)
sql.InitHistoryDatabase(true)
sql.InitAssetContentDatabase(true)
t.Cleanup(sql.CloseDatabase)
dailyPath, existed, err := CreateDailyNote(box.ID)
if nil != err {
t.Fatalf("create daily note failed: %v", err)
}
if existed {
t.Fatal("the first daily note creation should not reuse an existing document")
}
dailyID := util.GetTreeID(dailyPath)
assertTemplateOutputIndexed(t, dailyID)
boxConf.DailyNoteSavePath = "/Daily note/{{now | date \"2006-01-02\"}}"
boxConf.DailyNoteTemplatePath = "/dated.md"
if err := box.SaveConf(boxConf); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(templateDir, "dated.md"), []byte(`.action{now | date "2006-01-02"}`), 0644); err != nil {
t.Fatal(err)
}
target := time.Date(2024, 2, 29, 0, 0, 0, 0, time.Local)
info, err := GetDailyNoteInfo(box.ID, target)
if err != nil || info.Existed || info.ID != "" || info.HPath != "/Daily note/2024-02-29" || info.Title != "2024-02-29" {
t.Fatalf("unexpected missing daily note preview: %#v %v", info, err)
}
if treenode.GetBlockTreeRootByHPath(box.ID, info.HPath) != nil {
t.Fatal("preview created a document")
}
datedPath, existed, err := CreateDailyNoteAt(box.ID, target)
if err != nil || existed {
t.Fatalf("create dated note: %s %v %v", datedPath, existed, err)
}
datedID := util.GetTreeID(datedPath)
datedTree, err := LoadTreeByBlockID(datedID)
if err != nil {
t.Fatal(err)
}
if datedTree.HPath != "/Daily note/2024-02-29" || datedTree.Root.IALAttr(DailyNoteAttrPrefix+"20240229") != "20240229" {
t.Fatalf("date did not reach path and attribute: %#v", datedTree)
}
if content := string(datedTree.Root.Text()); !strings.Contains(content, "2024-02-29") {
t.Fatalf("content template used another date: %s", content)
}
if strings.HasPrefix(datedID, "20240229") || strings.HasPrefix(datedTree.Root.IALAttr("updated"), "20240229") {
t.Fatal("target date replaced real document timestamps")
}
reusedPath, existed, err := CreateDailyNoteAt(box.ID, target)
if err != nil || !existed || reusedPath != datedPath {
t.Fatalf("dated note was duplicated: %s %v %v", reusedPath, existed, err)
}
if err := RenameDoc(box.ID, datedPath, "Renamed daily note"); err != nil {
t.Fatal(err)
}
reusedPath, existed, err = CreateDailyNoteAt(box.ID, target)
if err != nil || !existed || reusedPath != datedPath {
t.Fatalf("renaming duplicated an existing date: %s %v %v", reusedPath, existed, err)
}
if err := MoveDocs([]string{box.ID + datedPath}, box.ID, "/", nil); err != nil {
t.Fatal(err)
}
datedPath = "/" + datedID + ".sy"
reusedPath, existed, err = CreateDailyNoteAt(box.ID, target)
if err != nil || !existed || reusedPath != datedPath {
t.Fatalf("moving duplicated an existing date: %s %v %v", reusedPath, existed, err)
}
boxConf.DailyNoteSavePath = "/Changed daily note/{{now | date \"2006-01-02\"}}"
if err := box.SaveConf(boxConf); err != nil {
t.Fatal(err)
}
reusedPath, existed, err = CreateDailyNoteAt(box.ID, target)
if err != nil || !existed || reusedPath != datedPath {
t.Fatalf("template change duplicated an existing date: %s %v %v", reusedPath, existed, err)
}
info, err = GetDailyNoteInfo(box.ID, target)
if err != nil || !info.Existed || info.ID != datedID || info.HPath != "/Renamed daily note" || info.Title != "Renamed daily note" {
t.Fatalf("preview did not reuse actual renamed and moved document: %#v %v", info, err)
}
beforeTree, err := LoadTreeByBlockID(datedID)
if err != nil {
t.Fatal(err)
}
beforeUpdated := beforeTree.Root.IALAttr("updated")
if _, err := GetDailyNoteInfo(box.ID, target); err != nil {
t.Fatal(err)
}
afterTree, err := LoadTreeByBlockID(datedID)
if err != nil {
t.Fatal(err)
}
if beforeUpdated != afterTree.Root.IALAttr("updated") {
t.Fatal("preview changed document timestamps")
}
docID := "20260728000001-abcdefg"
created, err := CreateDocByParent(DocumentCreateRequest{Notebook: box.ID, ID: docID,
TitleFormat: "plain", Title: "Default template", DocCreateTemplatePath: "/indexed.md"})
if nil != err {
t.Fatalf("create document with default template failed: %v", err)
}
if docID != created.ID {
t.Fatalf("unexpected created document ID: got %s, want %s", created.ID, docID)
}
assertTemplateOutputIndexed(t, docID)
attributeViewDocID := "20260728000002-abcdefg"
created, err = CreateDocByParent(DocumentCreateRequest{Notebook: box.ID, ID: attributeViewDocID,
TitleFormat: "plain", Title: "Attribute view template"})
if nil != err {
t.Fatalf("create attribute view item document failed: %v", err)
}
if err = applyNewItemContentTemplate("/indexed.md", created.ID); nil != err {
t.Fatalf("apply attribute view item content template failed: %v", err)
}
assertTemplateOutputIndexed(t, attributeViewDocID)
}
func assertTemplateOutputIndexed(t *testing.T, docID string) {
t.Helper()
blocks := sql.SelectBlocksRawStmt("SELECT * FROM blocks WHERE root_id = '"+docID+"' AND type = 'p'", 1, 32)
for _, block := range blocks {
if docID == block.Content {
return
}
}
t.Fatalf("template did not query and persist the indexed document [%s]", docID)
}
func assertDatabaseBlockExists(t *testing.T, id string, expected bool) {
t.Helper()
rows, err := sql.Query("SELECT id FROM blocks WHERE id = '"+id+"'", 1)
if nil != err {
t.Fatalf("query block [%s] failed: %v", id, err)
}
if actual := 0 < len(rows); actual != expected {
t.Fatalf("unexpected database block existence for [%s]: got %t, want %t", id, actual, expected)
}
}