Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:sparkles: Capture diagnostics for stalled deletion and indexing https://github.com/siyuan-note/siyuan/issues/20080

Danielec3f6189bb11
diff --git a/kernel/model/delete_stress_fts5_test.go b/kernel/model/delete_stress_fts5_test.go
new file mode 100644
index 0000000000..f141a8416e
--- /dev/null
+++ b/kernel/model/delete_stress_fts5_test.go
@@ -0,0 +1,177 @@
+//go:build fts5
+
+package model
+
+import (
+ "database/sql"
+ "os"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/88250/lute/ast"
+ "github.com/88250/lute/parse"
+ "github.com/siyuan-note/siyuan/kernel/conf"
+ index "github.com/siyuan-note/siyuan/kernel/sql"
+ "github.com/siyuan-note/siyuan/kernel/task"
+ "github.com/siyuan-note/siyuan/kernel/treenode"
+ "github.com/siyuan-note/siyuan/kernel/util"
+)
+
+// TestRapidDeletionLargeIndex 按需构造大索引,交叠运行删除、查询与后台任务;不创建等量的源文档。
+// SIYUAN_DELETE_STRESS_BLOCKS 指定填充块数,未设置时跳过,避免普通回归测试产生大量磁盘写入。
+func TestRapidDeletionLargeIndex(t *testing.T) {
+ value := os.Getenv("SIYUAN_DELETE_STRESS_BLOCKS")
+ if value == "" {
+ t.Skip("set SIYUAN_DELETE_STRESS_BLOCKS to run the large index diagnostic")
+ }
+ count, err := strconv.Atoi(value)
+ if err != nil || count < 1 || count > 2000000 {
+ t.Fatal("SIYUAN_DELETE_STRESS_BLOCKS must be between 1 and 2000000")
+ }
+ f := setupStructureTransactionTest(t)
+ setupFoldTransactionDatabase(t, f)
+ Conf.Search = conf.NewSearch()
+ source, err := LoadTreeByBlockID(f.sourceID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var blockIDs []string
+ for range 30 {
+ p := treenode.NewParagraph(ast.NewNodeID())
+ p.AppendChild(&ast.Node{Type: ast.NodeText, Tokens: []byte("delete text")})
+ source.Root.AppendChild(p)
+ blockIDs = append(blockIDs, p.ID)
+ }
+ if err = indexWriteTreeIndexQueue(source); err != nil {
+ t.Fatal(err)
+ }
+ var targets []*parse.Tree
+ for range 30 {
+ tree := treenode.NewTree(f.box.ID, "/"+ast.NewNodeID()+".sy", "/Delete target", "Delete target")
+ tree.Root.FirstChild.AppendChild(&ast.Node{Type: ast.NodeTextMark, TextMarkType: "block-ref",
+ TextMarkBlockRefID: f.childID, TextMarkBlockRefSubtype: "s", TextMarkTextContent: "Reference"})
+ if err = indexWriteTreeIndexQueue(tree); err != nil {
+ t.Fatal(err)
+ }
+ child := treenode.NewTree(f.box.ID, strings.TrimSuffix(tree.Path, ".sy")+"/"+ast.NewNodeID()+".sy", "/Delete target/Child", "Child")
+ if err = indexWriteTreeIndexQueue(child); err != nil {
+ t.Fatal(err)
+ }
+ targets = append(targets, tree)
+ }
+ index.FlushQueue()
+ seedStart := time.Now()
+ db, err := sql.Open("sqlite3_extended", util.DBPath+"?_journal_mode=WAL&_synchronous=OFF&_busy_timeout=7000")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ // 填充记录只用于复现扫描成本,真实删除目标仍由文档写入与索引入口创建。
+ _, err = db.Exec(`WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x < ?)
+ INSERT INTO blocks (id,parent_id,root_id,box,path,hpath,type,content,fcontent,markdown,ial)
+ SELECT printf('seed-%d',x),printf('seed-root-%d',x/30),printf('seed-root-%d',x/30),?,
+ printf('/seed/%d.sy',x/30),'/Seed','p',?,'','','' FROM n`, count, f.box.ID, strings.Repeat("content ", 64))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, statement := range []string{
+ "INSERT INTO attributes (id,name,value,root_id,box,path) SELECT id,'custom-test','value',root_id,box,path FROM blocks WHERE id LIKE 'seed-%'",
+ "INSERT INTO spans (id,root_id,box,path,content) SELECT id,root_id,box,path,'span' FROM blocks WHERE id LIKE 'seed-%' AND rowid%6=0",
+ "INSERT INTO refs (id,def_block_id,def_block_root_id,root_id,box,path) SELECT id,'seed-definition','seed-definition',root_id,box,path FROM blocks WHERE id LIKE 'seed-%' AND rowid%10=0",
+ "INSERT INTO assets (id,root_id,box,docpath,path) SELECT id,root_id,box,path,'assets/test.png' FROM blocks WHERE id LIKE 'seed-%' AND rowid%30=0",
+ "INSERT INTO blocks_fts(blocks_fts) VALUES('rebuild')",
+ } {
+ start := time.Now()
+ if _, err = db.Exec(statement); err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("seed stage %s: %s", strings.Fields(statement)[2], time.Since(start))
+ }
+ if _, err = db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {
+ t.Fatal(err)
+ }
+ info, err := os.Stat(util.DBPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("seeded %d blocks, database %.1f MiB, elapsed %s", count, float64(info.Size())/1048576, time.Since(seedStart))
+
+ stop := make(chan struct{})
+ var workers sync.WaitGroup
+ startWorker := func(interval time.Duration, run func()) {
+ workers.Add(1)
+ go func() {
+ defer workers.Done()
+ ticker := time.NewTicker(interval)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-stop:
+ return
+ case <-ticker.C:
+ run()
+ }
+ }
+ }()
+ }
+ startWorker(100*time.Millisecond, task.ExecTaskJob)
+ startWorker(100*time.Millisecond, task.ExecAsyncTaskJob)
+ startWorker(util.SQLFlushInterval, index.FlushTxJob)
+ startWorker(100*time.Millisecond, func() { _, _ = GetDocInfo(f.sourceID) })
+ defer func() {
+ close(stop)
+ workers.Wait()
+ index.FlushQueue()
+ // 等待已经派发的短延迟引用计数任务结束后再释放测试数据库。
+ time.Sleep(time.Second)
+ }()
+ textDone := make(chan error, 1)
+ go func() {
+ for _, id := range blockIDs {
+ if err := PerformTxSync(&Transaction{DoOperations: []*Operation{{Action: "delete", ID: id}}}); err != nil {
+ textDone <- err
+ return
+ }
+ time.Sleep(100 * time.Millisecond)
+ }
+ textDone <- nil
+ }()
+ start := time.Now()
+ for i, tree := range targets {
+ deletionStart := time.Now()
+ if err := RemoveDoc(f.box.ID, tree.Path); err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("delete %d: %s", i+1, time.Since(deletionStart))
+ if (i+1)%3 == 0 {
+ logStart := time.Now()
+ if err := writeSystemGoroutineLog(util.TempDir); err != nil {
+ t.Fatal(err)
+ }
+ t.Logf("stack capture: %s", time.Since(logStart))
+ }
+ }
+ if err := <-textDone; err != nil {
+ t.Fatal(err)
+ }
+ FlushTxQueue()
+ flushStart := time.Now()
+ index.FlushQueue()
+ t.Logf("final flush: %s, total deletion time: %s", time.Since(flushStart), time.Since(start))
+ for _, tree := range targets {
+ assertDatabaseBlockExists(t, tree.ID, false)
+ if treenode.GetBlockTree(tree.ID) != nil {
+ t.Fatal("deleted document remains in block tree")
+ }
+ }
+ var remaining int
+ if err := db.QueryRow("SELECT COUNT(*) FROM blocks WHERE id LIKE 'seed-%'").Scan(&remaining); err != nil {
+ t.Fatal(err)
+ }
+ if remaining != count {
+ t.Fatalf("unrelated blocks changed: got %d, want %d", remaining, count)
+ }
+}
diff --git a/kernel/model/export.go b/kernel/model/export.go
index 091611826f..5ea2a390ae 100644
--- a/kernel/model/export.go
+++ b/kernel/model/export.go
@@ -541,6 +541,8 @@ func ExportSystemLog() (zipPath string) {
if err := writeSystemGoroutineLog(exportFolder); err != nil {
logging.LogErrorf("export goroutine log failed: %s", err)
}
+ copyOptionalSystemLog(filepath.Join(util.TempDir, util.OperationStallLogName),
+ filepath.Join(exportFolder, util.OperationStallLogName), 0)
appLog := filepath.Join(util.HomeDir, ".config", "siyuan", "app.log")
if gulu.File.IsExist(appLog) {
diff --git a/kernel/model/export_system_log_test.go b/kernel/model/export_system_log_test.go
index c16690b992..2327b9d6ac 100644
--- a/kernel/model/export_system_log_test.go
+++ b/kernel/model/export_system_log_test.go
@@ -1,12 +1,96 @@
package model
import (
+ "archive/zip"
+ "io"
"os"
"path/filepath"
"strings"
"testing"
+ "time"
+
+ "github.com/siyuan-note/siyuan/kernel/util"
)
+func TestBlockedTransactionWaitSavesDiagnostic(t *testing.T) {
+ oldTemp := util.TempDir
+ util.TempDir = t.TempDir()
+ t.Cleanup(func() { util.TempDir = oldTemp })
+ flushLock.Lock()
+ isFlushing.Store(true)
+ finished := make(chan struct{})
+ go func() { FlushTxQueue(); close(finished) }()
+ defer func() {
+ isFlushing.Store(false)
+ flushLock.Unlock()
+ <-finished
+ }()
+ deadline := time.Now().Add(20 * time.Second)
+ for {
+ data, err := os.ReadFile(filepath.Join(util.TempDir, util.OperationStallLogName))
+ if err == nil {
+ for _, want := range []string{"flush editing transactions", "wait for transaction queue", "model.FlushTxQueue"} {
+ if !strings.Contains(string(data), want) {
+ t.Fatalf("diagnostic missing %q", want)
+ }
+ }
+ select {
+ case <-finished:
+ t.Fatal("transaction wait was bypassed")
+ default:
+ }
+ return
+ }
+ if !os.IsNotExist(err) {
+ t.Fatal(err)
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("blocked transaction did not save a diagnostic")
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+}
+
+func TestExportSystemLogIncludesStallWhileTransactionsBlocked(t *testing.T) {
+ oldTemp, oldHome, oldSystemTemp := util.TempDir, util.HomeDir, util.SystemTempDir
+ util.TempDir, util.HomeDir, util.SystemTempDir = t.TempDir(), t.TempDir(), t.TempDir()
+ t.Cleanup(func() {
+ util.TempDir, util.HomeDir, util.SystemTempDir = oldTemp, oldHome, oldSystemTemp
+ })
+ want := "operation: remove document\nstage: index document history\n"
+ if err := os.WriteFile(filepath.Join(util.TempDir, util.OperationStallLogName), []byte(want), 0600); err != nil {
+ t.Fatal(err)
+ }
+ // 持有事务执行锁并标记正在刷新,验证导出不依赖编辑事务完成。
+ flushLock.Lock()
+ isFlushing.Store(true)
+ defer func() { isFlushing.Store(false); flushLock.Unlock() }()
+ if exported := ExportSystemLog(); exported == "" {
+ t.Fatal("export failed")
+ }
+ archive, err := zip.OpenReader(filepath.Join(util.TempDir, "export", "system-log.zip"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer archive.Close()
+ for _, entry := range archive.File {
+ if filepath.Base(entry.Name) != util.OperationStallLogName {
+ continue
+ }
+ reader, err := entry.Open()
+ if err != nil {
+ t.Fatal(err)
+ }
+ data, err := io.ReadAll(reader)
+ reader.Close()
+ if err != nil || string(data) != want {
+ t.Fatalf("unexpected stall snapshot: %q, %v", data, err)
+ }
+ return
+ }
+ t.Fatal("stall snapshot missing from export")
+}
+
func TestWriteSystemGoroutineLog(t *testing.T) {
started, release := make(chan struct{}), make(chan struct{})
go waitForSystemLogTest(started, release)
diff --git a/kernel/model/file.go b/kernel/model/file.go
index e3020a5d32..46c302b96d 100644
--- a/kernel/model/file.go
+++ b/kernel/model/file.go
@@ -2078,10 +2078,13 @@ func RemoveDocs(paths []string) error {
}
func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err error) {
+ diagnostic := util.WatchOperation("remove document", "load document")
+ defer diagnostic.Finish()
ret, err = filesys.LoadTree(box.ID, p, luteEngine)
if err != nil || nil == ret {
return nil, ErrBlockNotFound
}
+ diagnostic.Stage("prepare local assets")
if !IsEncryptedBox(box.ID) {
assetParent := filepath.Dir(filepath.Join(util.DataDir, box.ID, p))
if err = EnsureAssetPrefixLocal(assetParent); err != nil {
@@ -2089,6 +2092,7 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err
}
}
+ diagnostic.Stage("backup document history")
historyDir, err := getHistoryDir(HistoryOpDelete)
if err != nil {
logging.LogErrorf("get history dir failed: %s", err)
@@ -2103,12 +2107,14 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err
}
// 加密笔记本的 assets 不提升到全局
+ diagnostic.Stage("copy document assets")
if !IsEncryptedBox(box.ID) {
if err = copyDocAssetsToDataAssets(box.ID, p); err != nil {
return nil, err
}
}
+ diagnostic.Stage("collect child documents")
removeIDs := treenode.RootChildIDs(ret.ID)
dir := path.Dir(p)
childrenDir := path.Join(dir, ret.ID)
@@ -2121,6 +2127,7 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err
}
existChildren := box.Exist(childrenDir)
if existChildren {
+ diagnostic.Stage("backup child document history")
absChildrenDir := filepath.Join(util.DataDir, ret.Box, childrenDir)
historyPath = filepath.Join(historyDir, ret.Box, childrenDir)
if err = filelock.Copy(absChildrenDir, historyPath); err != nil {
@@ -2128,6 +2135,7 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err
return
}
}
+ diagnostic.Stage("backup database bindings")
allRemoveRootIDs := []string{ret.ID}
allRemoveRootIDs = append(allRemoveRootIDs, removeIDs...)
allRemoveRootIDs = gulu.Str.RemoveDuplicatedElem(allRemoveRootIDs)
@@ -2146,12 +2154,15 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err
}
removeTrees = append(removeTrees, removeTree)
}
+ diagnostic.Stage("index document history")
indexHistoryDir(filepath.Base(historyDir), util.NewLute())
+ diagnostic.Stage("remove database bindings")
for _, removeTree := range removeTrees {
removedRootPaths[removeTree.ID] = removeTree.Path
syncDelete2AvBlock(removeTree.Root, removeTree, true, nil)
}
+ diagnostic.Stage("remove document files")
if existChildren {
if err = box.Remove(childrenDir); err != nil {
logging.LogErrorf("remove children dir [%s%s] failed: %s", box.ID, childrenDir, err)
@@ -2168,8 +2179,10 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err
for rootID := range removedRootPaths {
removedPins[rootID] = true
}
+ diagnostic.Stage("update pinned documents")
maintainPinnedDocs(removedPins, "", "")
+ diagnostic.Stage("update document order")
box.removeSort(removeIDs)
if "/" != dir {
others, err := os.ReadDir(filepath.Join(util.DataDir, box.ID, dir))
@@ -2178,6 +2191,7 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err
}
}
+ diagnostic.Stage("invalidate document caches")
for rootID, treePath := range removedRootPaths {
cache.RemoveTreeData(rootID)
cache.RemoveDocIAL(treePath)
@@ -2185,9 +2199,12 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err
for _, blockTree := range removedBlockTrees {
cache.RemoveBlockIAL(blockTree.ID)
}
+ diagnostic.Stage("remove block tree index")
treenode.RemoveBlockTreesByPathPrefix(box.ID, childrenDir)
+ diagnostic.Stage("queue database index deletion")
sql.RemoveTreePathQueue(ret.Box, childrenDir)
+ diagnostic.Stage("notify document deletion")
evt := util.NewCmdResult("removeDoc", 0, util.PushModeBroadcast)
evt.Data = map[string]any{
"ids": removeIDs,
diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go
index 449552d0e0..8b34140909 100644
--- a/kernel/model/transaction.go
+++ b/kernel/model/transaction.go
@@ -58,6 +58,8 @@ func IsMoveOutlineHeading(transactions *[]*Transaction) bool {
}
func FlushTxQueue() {
+ diagnostic := util.WatchOperation("flush editing transactions", "wait for transaction queue")
+ defer diagnostic.Finish()
time.Sleep(time.Duration(50) * time.Millisecond)
for 0 < txQueueSize() || isFlushing.Load() {
time.Sleep(10 * time.Millisecond)
@@ -225,6 +227,8 @@ func performTx(tx *Transaction) (ret *TxErr) {
if 1 > len(tx.DoOperations) {
return
}
+ diagnostic := util.WatchOperation("editing transaction", "begin transaction")
+ defer diagnostic.Finish()
//os.MkdirAll("pprof", 0755)
//cpuProfile, _ := os.Create("pprof/cpu_profile_tx")
@@ -254,15 +258,18 @@ func performTx(tx *Transaction) (ret *TxErr) {
}
}()
+ diagnostic.Stage("prepare database automations")
if err = tx.prepareAttributeViewAutomations(); err != nil {
tx.rollback()
return &TxErr{code: TxErrHandleAttributeView, msg: err.Error()}
}
+ diagnostic.Stage("process bulk operations")
isLargeInsert := tx.processLargeInsert()
isLargeDelete := tx.processLargeDelete()
if !isLargeInsert {
for operationIndex := 0; operationIndex < len(tx.DoOperations); operationIndex++ {
op := tx.DoOperations[operationIndex]
+ diagnostic.Stage("execute " + op.Action)
if isLargeDelete && "delete" == op.Action {
continue
}
@@ -498,6 +505,7 @@ func performTx(tx *Transaction) (ret *TxErr) {
}
}
+ diagnostic.Stage("finalize database bindings and block structure")
if err = tx.runAttributeViewAutomations(); err != nil {
tx.rollback()
return &TxErr{code: TxErrHandleAttributeView, msg: err.Error()}
@@ -516,6 +524,7 @@ func performTx(tx *Transaction) (ret *TxErr) {
}
tx.UndoOperations = append(tx.UndoOperations, tx.attributeViewDeletionUndo...)
+ diagnostic.Stage("commit transaction")
if cr := tx.commit(); nil != cr {
logging.LogErrorf("commit tx failed: %s", cr)
if 1 == tx.state.Load() {
diff --git a/kernel/sql/queue.go b/kernel/sql/queue.go
index b0772017f0..d75537e851 100644
--- a/kernel/sql/queue.go
+++ b/kernel/sql/queue.go
@@ -216,9 +216,12 @@ func clearQueue() {
var flushingTx = atomic.Bool{}
func FlushQueue() {
+ diagnostic := util.WatchOperation("flush database index", "wait for index lock")
+ defer diagnostic.Finish()
initDatabaseLock.Lock()
defer initDatabaseLock.Unlock()
+ diagnostic.Stage("take queued operations")
ops, indexSnapshot := getOperations()
total := len(ops)
if 1 > total && !flushingTx.Load() {
@@ -290,6 +293,7 @@ func FlushQueue() {
}
}
+ diagnostic.Stage("begin " + op.action)
tx, err := beginTxForBox(op.boxID())
if err != nil {
logging.LogWarnf("skip queue operation [%s] for box [%s]: %s", op.action, op.boxID(), err)
@@ -309,6 +313,7 @@ func FlushQueue() {
groupOpsCurrent[op.action]++
context["current"] = groupOpsCurrent[op.action]
context["total"] = groupOpsTotal[op.action]
+ diagnostic.Stage("execute " + op.action)
if err = execOp(op, tx, context); err != nil {
tx.Rollback()
closeTxPreparedStmts(tx)
@@ -321,6 +326,7 @@ func FlushQueue() {
continue
}
+ diagnostic.Stage("commit " + op.action)
if err = commitTx(tx); err != nil {
invalidateRefsCacheForOperation(op)
logging.LogErrorf("commit tx failed: %s", err)
@@ -362,11 +368,13 @@ func FlushQueue() {
}
// Push database index commit event https://github.com/siyuan-note/siyuan/issues/8814
+ diagnostic.Stage("notify index completion")
util.BroadcastByType("main", "databaseIndexCommit", 0, "", backlinkChange.data())
eventbus.Publish(eventbus.EvtSQLIndexFlushed)
// 刷新期间追加的操作仍在内存队列中,磁盘队列仅用于进程重启恢复,不能在这里重复执行。
+ diagnostic.Stage("persist index queue completion")
clearIndexQueue(indexSnapshot)
}
diff --git a/kernel/util/operation_watchdog.go b/kernel/util/operation_watchdog.go
new file mode 100644
index 0000000000..3f13a28886
--- /dev/null
+++ b/kernel/util/operation_watchdog.go
@@ -0,0 +1,116 @@
+package util
+
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "runtime"
+ "sync"
+ "time"
+
+ "github.com/siyuan-note/logging"
+)
+
+const OperationStallLogName = "operation-stall.log"
+
+const operationStallThreshold = 10 * time.Second
+
+// OperationWatchdog 在操作仍未返回时记录阶段和调用栈,不取消操作、不改变数据写入顺序。
+type OperationWatchdog struct {
+ mu sync.Mutex
+ name string
+ stage string
+ started time.Time
+ done bool
+ timer *time.Timer
+}
+
+type operationStall struct {
+ name, stage string
+ elapsed time.Duration
+}
+
+func WatchOperation(name, stage string) *OperationWatchdog {
+ dir := TempDir
+ return watchOperation(name, stage, operationStallThreshold, func(stall operationStall) {
+ // 先保存独立文件,再写常规日志,避免常规日志写入阻塞时丢失现场。
+ operationStallRecorder.record(dir, stall)
+ logging.LogWarnf("operation still running [%s], stage [%s], elapsed [%s]", stall.name, stall.stage, stall.elapsed)
+ })
+}
+
+func watchOperation(name, stage string, threshold time.Duration, report func(operationStall)) *OperationWatchdog {
+ w := &OperationWatchdog{name: name, stage: stage, started: time.Now()}
+ w.timer = time.AfterFunc(threshold, func() {
+ w.mu.Lock()
+ if w.done {
+ w.mu.Unlock()
+ return
+ }
+ stall := operationStall{w.name, w.stage, time.Since(w.started)}
+ w.mu.Unlock()
+ report(stall)
+ })
+ return w
+}
+
+func (w *OperationWatchdog) Stage(stage string) {
+ w.mu.Lock()
+ w.stage = stage
+ w.mu.Unlock()
+}
+
+func (w *OperationWatchdog) Finish() {
+ w.mu.Lock()
+ w.done = true
+ w.timer.Stop()
+ elapsed, stage := time.Since(w.started), w.stage
+ w.mu.Unlock()
+ if elapsed >= operationStallThreshold {
+ logging.LogInfof("slow operation finished [%s], stage [%s], elapsed [%s]", w.name, stage, elapsed)
+ }
+}
+
+type stallRecorder struct {
+ mu sync.Mutex
+ last time.Time
+}
+
+var operationStallRecorder stallRecorder
+
+// record 每分钟最多保存一次,覆盖上次现场;快照最多 8 MiB,不持有业务锁。
+func (r *stallRecorder) record(dir string, stall operationStall) {
+ if dir == "" || !r.mu.TryLock() {
+ return
+ }
+ defer r.mu.Unlock()
+ if time.Since(r.last) < time.Minute {
+ return
+ }
+ buffer := make([]byte, 64*1024)
+ var n int
+ for {
+ n = runtime.Stack(buffer, true)
+ if n < len(buffer) || len(buffer) >= 8*1024*1024 {
+ break
+ }
+ buffer = make([]byte, len(buffer)*2)
+ }
+ header := fmt.Sprintf("time: %s\noperation: %s\nstage: %s\nelapsed: %s\ntruncated: %t\n\n",
+ time.Now().Format(time.RFC3339), stall.name, stall.stage, stall.elapsed, n == len(buffer))
+ // 用临时文件替换,避免导出读到正在写入的半份调用栈。
+ file, err := os.CreateTemp(dir, "operation-stall-*.tmp")
+ if err != nil {
+ return
+ }
+ tmp := file.Name()
+ defer os.Remove(tmp)
+ _, writeErr := file.Write(append([]byte(header), buffer[:n]...))
+ closeErr := file.Close()
+ if writeErr != nil || closeErr != nil {
+ return
+ }
+ if err = os.Rename(tmp, filepath.Join(dir, OperationStallLogName)); err == nil {
+ r.last = time.Now()
+ }
+}
diff --git a/kernel/util/operation_watchdog_test.go b/kernel/util/operation_watchdog_test.go
new file mode 100644
index 0000000000..df90df5f74
--- /dev/null
+++ b/kernel/util/operation_watchdog_test.go
@@ -0,0 +1,69 @@
+package util
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestOperationWatchdogCapturesBlockedStage(t *testing.T) {
+ reported := make(chan operationStall, 1)
+ w := watchOperation("delete", "start", 20*time.Millisecond, func(stall operationStall) { reported <- stall })
+ defer w.Finish()
+ w.Stage("waiting for index")
+ select {
+ case stall := <-reported:
+ if stall.name != "delete" || stall.stage != "waiting for index" || stall.elapsed < 20*time.Millisecond {
+ t.Fatalf("unexpected diagnostic: %+v", stall)
+ }
+ case <-time.After(2 * time.Second):
+ t.Fatal("blocked operation was not reported before completion")
+ }
+}
+
+func TestOperationWatchdogStopsAfterCompletion(t *testing.T) {
+ reported := make(chan operationStall, 1)
+ w := watchOperation("delete", "start", 20*time.Millisecond, func(stall operationStall) { reported <- stall })
+ w.Finish()
+ select {
+ case <-reported:
+ t.Fatal("completed operation was reported as blocked")
+ case <-time.After(50 * time.Millisecond):
+ }
+}
+
+func TestOperationStallSnapshotIsBoundedAndRateLimited(t *testing.T) {
+ dir := t.TempDir()
+ r := &stallRecorder{}
+ r.record(dir, operationStall{"delete", "index", 10 * time.Second})
+ path := filepath.Join(dir, OperationStallLogName)
+ data, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, want := range []string{"operation: delete", "stage: index", "goroutine ", "TestOperationStallSnapshot"} {
+ if !strings.Contains(string(data), want) {
+ t.Fatalf("missing %q", want)
+ }
+ }
+ if len(data) > 8*1024*1024+1024 {
+ t.Fatal("snapshot exceeds size limit")
+ }
+ r.record(dir, operationStall{"second", "index", 10 * time.Second})
+ again, err := os.ReadFile(path)
+ if err != nil || string(data) != string(again) {
+ t.Fatal("rate limit did not preserve first snapshot")
+ }
+ r.last = time.Now().Add(-2 * time.Minute)
+ r.record(dir, operationStall{"third", "index", 10 * time.Second})
+ again, err = os.ReadFile(path)
+ if err != nil || !strings.Contains(string(again), "operation: third") {
+ t.Fatal("snapshot was not replaced after rate limit expired")
+ }
+ files, err := os.ReadDir(dir)
+ if err != nil || len(files) != 1 {
+ t.Fatal("temporary diagnostic files remain")
+ }
+}