An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作
TypeScript agentic-aiai-agentknowledge-baselocal-firstmarkdownmcpmindmapnote-takingself-hostedsiyuanworkspace
:sparkles: Capture diagnostics for stalled deletion and indexing https://github.com/siyuan-note/siyuan/issues/20080
Danielec3f6189bb11
diff --git a/kernel/model/delete_stress_fts5_test.go b/kernel/model/delete_stress_fts5_test.gonew file mode 100644index 0000000000..f141a8416e--- /dev/null+++ b/kernel/model/delete_stress_fts5_test.go@@ -0,0 +1,177 @@+//go:build fts5++package model++import (+ "database/sql"+ "os"+ "strconv"+ "strings"+ "sync"+ "testing"+ "time"++ "github.com/88250/lute/ast"+ "github.com/88250/lute/parse"+ "github.com/siyuan-note/siyuan/kernel/conf"+ index "github.com/siyuan-note/siyuan/kernel/sql"+ "github.com/siyuan-note/siyuan/kernel/task"+ "github.com/siyuan-note/siyuan/kernel/treenode"+ "github.com/siyuan-note/siyuan/kernel/util"+)++// TestRapidDeletionLargeIndex 按需构造大索引,交叠运行删除、查询与后台任务;不创建等量的源文档。+// SIYUAN_DELETE_STRESS_BLOCKS 指定填充块数,未设置时跳过,避免普通回归测试产生大量磁盘写入。+func TestRapidDeletionLargeIndex(t *testing.T) {+ value := os.Getenv("SIYUAN_DELETE_STRESS_BLOCKS")+ if value == "" {+ t.Skip("set SIYUAN_DELETE_STRESS_BLOCKS to run the large index diagnostic")+ }+ count, err := strconv.Atoi(value)+ if err != nil || count < 1 || count > 2000000 {+ t.Fatal("SIYUAN_DELETE_STRESS_BLOCKS must be between 1 and 2000000")+ }+ f := setupStructureTransactionTest(t)+ setupFoldTransactionDatabase(t, f)+ Conf.Search = conf.NewSearch()+ source, err := LoadTreeByBlockID(f.sourceID)+ if err != nil {+ t.Fatal(err)+ }+ var blockIDs []string+ for range 30 {+ p := treenode.NewParagraph(ast.NewNodeID())+ p.AppendChild(&ast.Node{Type: ast.NodeText, Tokens: []byte("delete text")})+ source.Root.AppendChild(p)+ blockIDs = append(blockIDs, p.ID)+ }+ if err = indexWriteTreeIndexQueue(source); err != nil {+ t.Fatal(err)+ }+ var targets []*parse.Tree+ for range 30 {+ tree := treenode.NewTree(f.box.ID, "/"+ast.NewNodeID()+".sy", "/Delete target", "Delete target")+ tree.Root.FirstChild.AppendChild(&ast.Node{Type: ast.NodeTextMark, TextMarkType: "block-ref",+ TextMarkBlockRefID: f.childID, TextMarkBlockRefSubtype: "s", TextMarkTextContent: "Reference"})+ if err = indexWriteTreeIndexQueue(tree); err != nil {+ t.Fatal(err)+ }+ child := treenode.NewTree(f.box.ID, strings.TrimSuffix(tree.Path, ".sy")+"/"+ast.NewNodeID()+".sy", "/Delete target/Child", "Child")+ if err = indexWriteTreeIndexQueue(child); err != nil {+ t.Fatal(err)+ }+ targets = append(targets, tree)+ }+ index.FlushQueue()+ seedStart := time.Now()+ db, err := sql.Open("sqlite3_extended", util.DBPath+"?_journal_mode=WAL&_synchronous=OFF&_busy_timeout=7000")+ if err != nil {+ t.Fatal(err)+ }+ defer db.Close()+ // 填充记录只用于复现扫描成本,真实删除目标仍由文档写入与索引入口创建。+ _, err = db.Exec(`WITH RECURSIVE n(x) AS (VALUES(1) UNION ALL SELECT x+1 FROM n WHERE x < ?)+ INSERT INTO blocks (id,parent_id,root_id,box,path,hpath,type,content,fcontent,markdown,ial)+ SELECT printf('seed-%d',x),printf('seed-root-%d',x/30),printf('seed-root-%d',x/30),?,+ printf('/seed/%d.sy',x/30),'/Seed','p',?,'','','' FROM n`, count, f.box.ID, strings.Repeat("content ", 64))+ if err != nil {+ t.Fatal(err)+ }+ for _, statement := range []string{+ "INSERT INTO attributes (id,name,value,root_id,box,path) SELECT id,'custom-test','value',root_id,box,path FROM blocks WHERE id LIKE 'seed-%'",+ "INSERT INTO spans (id,root_id,box,path,content) SELECT id,root_id,box,path,'span' FROM blocks WHERE id LIKE 'seed-%' AND rowid%6=0",+ "INSERT INTO refs (id,def_block_id,def_block_root_id,root_id,box,path) SELECT id,'seed-definition','seed-definition',root_id,box,path FROM blocks WHERE id LIKE 'seed-%' AND rowid%10=0",+ "INSERT INTO assets (id,root_id,box,docpath,path) SELECT id,root_id,box,path,'assets/test.png' FROM blocks WHERE id LIKE 'seed-%' AND rowid%30=0",+ "INSERT INTO blocks_fts(blocks_fts) VALUES('rebuild')",+ } {+ start := time.Now()+ if _, err = db.Exec(statement); err != nil {+ t.Fatal(err)+ }+ t.Logf("seed stage %s: %s", strings.Fields(statement)[2], time.Since(start))+ }+ if _, err = db.Exec("PRAGMA wal_checkpoint(TRUNCATE)"); err != nil {+ t.Fatal(err)+ }+ info, err := os.Stat(util.DBPath)+ if err != nil {+ t.Fatal(err)+ }+ t.Logf("seeded %d blocks, database %.1f MiB, elapsed %s", count, float64(info.Size())/1048576, time.Since(seedStart))++ stop := make(chan struct{})+ var workers sync.WaitGroup+ startWorker := func(interval time.Duration, run func()) {+ workers.Add(1)+ go func() {+ defer workers.Done()+ ticker := time.NewTicker(interval)+ defer ticker.Stop()+ for {+ select {+ case <-stop:+ return+ case <-ticker.C:+ run()+ }+ }+ }()+ }+ startWorker(100*time.Millisecond, task.ExecTaskJob)+ startWorker(100*time.Millisecond, task.ExecAsyncTaskJob)+ startWorker(util.SQLFlushInterval, index.FlushTxJob)+ startWorker(100*time.Millisecond, func() { _, _ = GetDocInfo(f.sourceID) })+ defer func() {+ close(stop)+ workers.Wait()+ index.FlushQueue()+ // 等待已经派发的短延迟引用计数任务结束后再释放测试数据库。+ time.Sleep(time.Second)+ }()+ textDone := make(chan error, 1)+ go func() {+ for _, id := range blockIDs {+ if err := PerformTxSync(&Transaction{DoOperations: []*Operation{{Action: "delete", ID: id}}}); err != nil {+ textDone <- err+ return+ }+ time.Sleep(100 * time.Millisecond)+ }+ textDone <- nil+ }()+ start := time.Now()+ for i, tree := range targets {+ deletionStart := time.Now()+ if err := RemoveDoc(f.box.ID, tree.Path); err != nil {+ t.Fatal(err)+ }+ t.Logf("delete %d: %s", i+1, time.Since(deletionStart))+ if (i+1)%3 == 0 {+ logStart := time.Now()+ if err := writeSystemGoroutineLog(util.TempDir); err != nil {+ t.Fatal(err)+ }+ t.Logf("stack capture: %s", time.Since(logStart))+ }+ }+ if err := <-textDone; err != nil {+ t.Fatal(err)+ }+ FlushTxQueue()+ flushStart := time.Now()+ index.FlushQueue()+ t.Logf("final flush: %s, total deletion time: %s", time.Since(flushStart), time.Since(start))+ for _, tree := range targets {+ assertDatabaseBlockExists(t, tree.ID, false)+ if treenode.GetBlockTree(tree.ID) != nil {+ t.Fatal("deleted document remains in block tree")+ }+ }+ var remaining int+ if err := db.QueryRow("SELECT COUNT(*) FROM blocks WHERE id LIKE 'seed-%'").Scan(&remaining); err != nil {+ t.Fatal(err)+ }+ if remaining != count {+ t.Fatalf("unrelated blocks changed: got %d, want %d", remaining, count)+ }+}diff --git a/kernel/model/export.go b/kernel/model/export.goindex 091611826f..5ea2a390ae 100644--- a/kernel/model/export.go+++ b/kernel/model/export.go@@ -541,6 +541,8 @@ func ExportSystemLog() (zipPath string) {if err := writeSystemGoroutineLog(exportFolder); err != nil {logging.LogErrorf("export goroutine log failed: %s", err)}+ copyOptionalSystemLog(filepath.Join(util.TempDir, util.OperationStallLogName),+ filepath.Join(exportFolder, util.OperationStallLogName), 0)appLog := filepath.Join(util.HomeDir, ".config", "siyuan", "app.log")if gulu.File.IsExist(appLog) {diff --git a/kernel/model/export_system_log_test.go b/kernel/model/export_system_log_test.goindex c16690b992..2327b9d6ac 100644--- a/kernel/model/export_system_log_test.go+++ b/kernel/model/export_system_log_test.go@@ -1,12 +1,96 @@package modelimport (+ "archive/zip"+ "io""os""path/filepath""strings""testing"+ "time"++ "github.com/siyuan-note/siyuan/kernel/util")+func TestBlockedTransactionWaitSavesDiagnostic(t *testing.T) {+ oldTemp := util.TempDir+ util.TempDir = t.TempDir()+ t.Cleanup(func() { util.TempDir = oldTemp })+ flushLock.Lock()+ isFlushing.Store(true)+ finished := make(chan struct{})+ go func() { FlushTxQueue(); close(finished) }()+ defer func() {+ isFlushing.Store(false)+ flushLock.Unlock()+ <-finished+ }()+ deadline := time.Now().Add(20 * time.Second)+ for {+ data, err := os.ReadFile(filepath.Join(util.TempDir, util.OperationStallLogName))+ if err == nil {+ for _, want := range []string{"flush editing transactions", "wait for transaction queue", "model.FlushTxQueue"} {+ if !strings.Contains(string(data), want) {+ t.Fatalf("diagnostic missing %q", want)+ }+ }+ select {+ case <-finished:+ t.Fatal("transaction wait was bypassed")+ default:+ }+ return+ }+ if !os.IsNotExist(err) {+ t.Fatal(err)+ }+ if time.Now().After(deadline) {+ t.Fatal("blocked transaction did not save a diagnostic")+ }+ time.Sleep(20 * time.Millisecond)+ }+}++func TestExportSystemLogIncludesStallWhileTransactionsBlocked(t *testing.T) {+ oldTemp, oldHome, oldSystemTemp := util.TempDir, util.HomeDir, util.SystemTempDir+ util.TempDir, util.HomeDir, util.SystemTempDir = t.TempDir(), t.TempDir(), t.TempDir()+ t.Cleanup(func() {+ util.TempDir, util.HomeDir, util.SystemTempDir = oldTemp, oldHome, oldSystemTemp+ })+ want := "operation: remove document\nstage: index document history\n"+ if err := os.WriteFile(filepath.Join(util.TempDir, util.OperationStallLogName), []byte(want), 0600); err != nil {+ t.Fatal(err)+ }+ // 持有事务执行锁并标记正在刷新,验证导出不依赖编辑事务完成。+ flushLock.Lock()+ isFlushing.Store(true)+ defer func() { isFlushing.Store(false); flushLock.Unlock() }()+ if exported := ExportSystemLog(); exported == "" {+ t.Fatal("export failed")+ }+ archive, err := zip.OpenReader(filepath.Join(util.TempDir, "export", "system-log.zip"))+ if err != nil {+ t.Fatal(err)+ }+ defer archive.Close()+ for _, entry := range archive.File {+ if filepath.Base(entry.Name) != util.OperationStallLogName {+ continue+ }+ reader, err := entry.Open()+ if err != nil {+ t.Fatal(err)+ }+ data, err := io.ReadAll(reader)+ reader.Close()+ if err != nil || string(data) != want {+ t.Fatalf("unexpected stall snapshot: %q, %v", data, err)+ }+ return+ }+ t.Fatal("stall snapshot missing from export")+}+func TestWriteSystemGoroutineLog(t *testing.T) {started, release := make(chan struct{}), make(chan struct{})go waitForSystemLogTest(started, release)diff --git a/kernel/model/file.go b/kernel/model/file.goindex e3020a5d32..46c302b96d 100644--- a/kernel/model/file.go+++ b/kernel/model/file.go@@ -2078,10 +2078,13 @@ func RemoveDocs(paths []string) error {}func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err error) {+ diagnostic := util.WatchOperation("remove document", "load document")+ defer diagnostic.Finish()ret, err = filesys.LoadTree(box.ID, p, luteEngine)if err != nil || nil == ret {return nil, ErrBlockNotFound}+ diagnostic.Stage("prepare local assets")if !IsEncryptedBox(box.ID) {assetParent := filepath.Dir(filepath.Join(util.DataDir, box.ID, p))if err = EnsureAssetPrefixLocal(assetParent); err != nil {@@ -2089,6 +2092,7 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err}}+ diagnostic.Stage("backup document history")historyDir, err := getHistoryDir(HistoryOpDelete)if err != nil {logging.LogErrorf("get history dir failed: %s", err)@@ -2103,12 +2107,14 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err}// 加密笔记本的 assets 不提升到全局+ diagnostic.Stage("copy document assets")if !IsEncryptedBox(box.ID) {if err = copyDocAssetsToDataAssets(box.ID, p); err != nil {return nil, err}}+ diagnostic.Stage("collect child documents")removeIDs := treenode.RootChildIDs(ret.ID)dir := path.Dir(p)childrenDir := path.Join(dir, ret.ID)@@ -2121,6 +2127,7 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err}existChildren := box.Exist(childrenDir)if existChildren {+ diagnostic.Stage("backup child document history")absChildrenDir := filepath.Join(util.DataDir, ret.Box, childrenDir)historyPath = filepath.Join(historyDir, ret.Box, childrenDir)if err = filelock.Copy(absChildrenDir, historyPath); err != nil {@@ -2128,6 +2135,7 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, errreturn}}+ diagnostic.Stage("backup database bindings")allRemoveRootIDs := []string{ret.ID}allRemoveRootIDs = append(allRemoveRootIDs, removeIDs...)allRemoveRootIDs = gulu.Str.RemoveDuplicatedElem(allRemoveRootIDs)@@ -2146,12 +2154,15 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err}removeTrees = append(removeTrees, removeTree)}+ diagnostic.Stage("index document history")indexHistoryDir(filepath.Base(historyDir), util.NewLute())+ diagnostic.Stage("remove database bindings")for _, removeTree := range removeTrees {removedRootPaths[removeTree.ID] = removeTree.PathsyncDelete2AvBlock(removeTree.Root, removeTree, true, nil)}+ diagnostic.Stage("remove document files")if existChildren {if err = box.Remove(childrenDir); err != nil {logging.LogErrorf("remove children dir [%s%s] failed: %s", box.ID, childrenDir, err)@@ -2168,8 +2179,10 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, errfor rootID := range removedRootPaths {removedPins[rootID] = true}+ diagnostic.Stage("update pinned documents")maintainPinnedDocs(removedPins, "", "")+ diagnostic.Stage("update document order")box.removeSort(removeIDs)if "/" != dir {others, err := os.ReadDir(filepath.Join(util.DataDir, box.ID, dir))@@ -2178,6 +2191,7 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, err}}+ diagnostic.Stage("invalidate document caches")for rootID, treePath := range removedRootPaths {cache.RemoveTreeData(rootID)cache.RemoveDocIAL(treePath)@@ -2185,9 +2199,12 @@ func removeDoc(box *Box, p string, luteEngine *lute.Lute) (ret *parse.Tree, errfor _, blockTree := range removedBlockTrees {cache.RemoveBlockIAL(blockTree.ID)}+ diagnostic.Stage("remove block tree index")treenode.RemoveBlockTreesByPathPrefix(box.ID, childrenDir)+ diagnostic.Stage("queue database index deletion")sql.RemoveTreePathQueue(ret.Box, childrenDir)+ diagnostic.Stage("notify document deletion")evt := util.NewCmdResult("removeDoc", 0, util.PushModeBroadcast)evt.Data = map[string]any{"ids": removeIDs,diff --git a/kernel/model/transaction.go b/kernel/model/transaction.goindex 449552d0e0..8b34140909 100644--- a/kernel/model/transaction.go+++ b/kernel/model/transaction.go@@ -58,6 +58,8 @@ func IsMoveOutlineHeading(transactions *[]*Transaction) bool {}func FlushTxQueue() {+ diagnostic := util.WatchOperation("flush editing transactions", "wait for transaction queue")+ defer diagnostic.Finish()time.Sleep(time.Duration(50) * time.Millisecond)for 0 < txQueueSize() || isFlushing.Load() {time.Sleep(10 * time.Millisecond)@@ -225,6 +227,8 @@ func performTx(tx *Transaction) (ret *TxErr) {if 1 > len(tx.DoOperations) {return}+ diagnostic := util.WatchOperation("editing transaction", "begin transaction")+ defer diagnostic.Finish()//os.MkdirAll("pprof", 0755)//cpuProfile, _ := os.Create("pprof/cpu_profile_tx")@@ -254,15 +258,18 @@ func performTx(tx *Transaction) (ret *TxErr) {}}()+ diagnostic.Stage("prepare database automations")if err = tx.prepareAttributeViewAutomations(); err != nil {tx.rollback()return &TxErr{code: TxErrHandleAttributeView, msg: err.Error()}}+ diagnostic.Stage("process bulk operations")isLargeInsert := tx.processLargeInsert()isLargeDelete := tx.processLargeDelete()if !isLargeInsert {for operationIndex := 0; operationIndex < len(tx.DoOperations); operationIndex++ {op := tx.DoOperations[operationIndex]+ diagnostic.Stage("execute " + op.Action)if isLargeDelete && "delete" == op.Action {continue}@@ -498,6 +505,7 @@ func performTx(tx *Transaction) (ret *TxErr) {}}+ diagnostic.Stage("finalize database bindings and block structure")if err = tx.runAttributeViewAutomations(); err != nil {tx.rollback()return &TxErr{code: TxErrHandleAttributeView, msg: err.Error()}@@ -516,6 +524,7 @@ func performTx(tx *Transaction) (ret *TxErr) {}tx.UndoOperations = append(tx.UndoOperations, tx.attributeViewDeletionUndo...)+ diagnostic.Stage("commit transaction")if cr := tx.commit(); nil != cr {logging.LogErrorf("commit tx failed: %s", cr)if 1 == tx.state.Load() {diff --git a/kernel/sql/queue.go b/kernel/sql/queue.goindex b0772017f0..d75537e851 100644--- a/kernel/sql/queue.go+++ b/kernel/sql/queue.go@@ -216,9 +216,12 @@ func clearQueue() {var flushingTx = atomic.Bool{}func FlushQueue() {+ diagnostic := util.WatchOperation("flush database index", "wait for index lock")+ defer diagnostic.Finish()initDatabaseLock.Lock()defer initDatabaseLock.Unlock()+ diagnostic.Stage("take queued operations")ops, indexSnapshot := getOperations()total := len(ops)if 1 > total && !flushingTx.Load() {@@ -290,6 +293,7 @@ func FlushQueue() {}}+ diagnostic.Stage("begin " + op.action)tx, err := beginTxForBox(op.boxID())if err != nil {logging.LogWarnf("skip queue operation [%s] for box [%s]: %s", op.action, op.boxID(), err)@@ -309,6 +313,7 @@ func FlushQueue() {groupOpsCurrent[op.action]++context["current"] = groupOpsCurrent[op.action]context["total"] = groupOpsTotal[op.action]+ diagnostic.Stage("execute " + op.action)if err = execOp(op, tx, context); err != nil {tx.Rollback()closeTxPreparedStmts(tx)@@ -321,6 +326,7 @@ func FlushQueue() {continue}+ diagnostic.Stage("commit " + op.action)if err = commitTx(tx); err != nil {invalidateRefsCacheForOperation(op)logging.LogErrorf("commit tx failed: %s", err)@@ -362,11 +368,13 @@ func FlushQueue() {}// Push database index commit event https://github.com/siyuan-note/siyuan/issues/8814+ diagnostic.Stage("notify index completion")util.BroadcastByType("main", "databaseIndexCommit", 0, "", backlinkChange.data())eventbus.Publish(eventbus.EvtSQLIndexFlushed)// 刷新期间追加的操作仍在内存队列中,磁盘队列仅用于进程重启恢复,不能在这里重复执行。+ diagnostic.Stage("persist index queue completion")clearIndexQueue(indexSnapshot)}diff --git a/kernel/util/operation_watchdog.go b/kernel/util/operation_watchdog.gonew file mode 100644index 0000000000..3f13a28886--- /dev/null+++ b/kernel/util/operation_watchdog.go@@ -0,0 +1,116 @@+package util++import (+ "fmt"+ "os"+ "path/filepath"+ "runtime"+ "sync"+ "time"++ "github.com/siyuan-note/logging"+)++const OperationStallLogName = "operation-stall.log"++const operationStallThreshold = 10 * time.Second++// OperationWatchdog 在操作仍未返回时记录阶段和调用栈,不取消操作、不改变数据写入顺序。+type OperationWatchdog struct {+ mu sync.Mutex+ name string+ stage string+ started time.Time+ done bool+ timer *time.Timer+}++type operationStall struct {+ name, stage string+ elapsed time.Duration+}++func WatchOperation(name, stage string) *OperationWatchdog {+ dir := TempDir+ return watchOperation(name, stage, operationStallThreshold, func(stall operationStall) {+ // 先保存独立文件,再写常规日志,避免常规日志写入阻塞时丢失现场。+ operationStallRecorder.record(dir, stall)+ logging.LogWarnf("operation still running [%s], stage [%s], elapsed [%s]", stall.name, stall.stage, stall.elapsed)+ })+}++func watchOperation(name, stage string, threshold time.Duration, report func(operationStall)) *OperationWatchdog {+ w := &OperationWatchdog{name: name, stage: stage, started: time.Now()}+ w.timer = time.AfterFunc(threshold, func() {+ w.mu.Lock()+ if w.done {+ w.mu.Unlock()+ return+ }+ stall := operationStall{w.name, w.stage, time.Since(w.started)}+ w.mu.Unlock()+ report(stall)+ })+ return w+}++func (w *OperationWatchdog) Stage(stage string) {+ w.mu.Lock()+ w.stage = stage+ w.mu.Unlock()+}++func (w *OperationWatchdog) Finish() {+ w.mu.Lock()+ w.done = true+ w.timer.Stop()+ elapsed, stage := time.Since(w.started), w.stage+ w.mu.Unlock()+ if elapsed >= operationStallThreshold {+ logging.LogInfof("slow operation finished [%s], stage [%s], elapsed [%s]", w.name, stage, elapsed)+ }+}++type stallRecorder struct {+ mu sync.Mutex+ last time.Time+}++var operationStallRecorder stallRecorder++// record 每分钟最多保存一次,覆盖上次现场;快照最多 8 MiB,不持有业务锁。+func (r *stallRecorder) record(dir string, stall operationStall) {+ if dir == "" || !r.mu.TryLock() {+ return+ }+ defer r.mu.Unlock()+ if time.Since(r.last) < time.Minute {+ return+ }+ buffer := make([]byte, 64*1024)+ var n int+ for {+ n = runtime.Stack(buffer, true)+ if n < len(buffer) || len(buffer) >= 8*1024*1024 {+ break+ }+ buffer = make([]byte, len(buffer)*2)+ }+ header := fmt.Sprintf("time: %s\noperation: %s\nstage: %s\nelapsed: %s\ntruncated: %t\n\n",+ time.Now().Format(time.RFC3339), stall.name, stall.stage, stall.elapsed, n == len(buffer))+ // 用临时文件替换,避免导出读到正在写入的半份调用栈。+ file, err := os.CreateTemp(dir, "operation-stall-*.tmp")+ if err != nil {+ return+ }+ tmp := file.Name()+ defer os.Remove(tmp)+ _, writeErr := file.Write(append([]byte(header), buffer[:n]...))+ closeErr := file.Close()+ if writeErr != nil || closeErr != nil {+ return+ }+ if err = os.Rename(tmp, filepath.Join(dir, OperationStallLogName)); err == nil {+ r.last = time.Now()+ }+}diff --git a/kernel/util/operation_watchdog_test.go b/kernel/util/operation_watchdog_test.gonew file mode 100644index 0000000000..df90df5f74--- /dev/null+++ b/kernel/util/operation_watchdog_test.go@@ -0,0 +1,69 @@+package util++import (+ "os"+ "path/filepath"+ "strings"+ "testing"+ "time"+)++func TestOperationWatchdogCapturesBlockedStage(t *testing.T) {+ reported := make(chan operationStall, 1)+ w := watchOperation("delete", "start", 20*time.Millisecond, func(stall operationStall) { reported <- stall })+ defer w.Finish()+ w.Stage("waiting for index")+ select {+ case stall := <-reported:+ if stall.name != "delete" || stall.stage != "waiting for index" || stall.elapsed < 20*time.Millisecond {+ t.Fatalf("unexpected diagnostic: %+v", stall)+ }+ case <-time.After(2 * time.Second):+ t.Fatal("blocked operation was not reported before completion")+ }+}++func TestOperationWatchdogStopsAfterCompletion(t *testing.T) {+ reported := make(chan operationStall, 1)+ w := watchOperation("delete", "start", 20*time.Millisecond, func(stall operationStall) { reported <- stall })+ w.Finish()+ select {+ case <-reported:+ t.Fatal("completed operation was reported as blocked")+ case <-time.After(50 * time.Millisecond):+ }+}++func TestOperationStallSnapshotIsBoundedAndRateLimited(t *testing.T) {+ dir := t.TempDir()+ r := &stallRecorder{}+ r.record(dir, operationStall{"delete", "index", 10 * time.Second})+ path := filepath.Join(dir, OperationStallLogName)+ data, err := os.ReadFile(path)+ if err != nil {+ t.Fatal(err)+ }+ for _, want := range []string{"operation: delete", "stage: index", "goroutine ", "TestOperationStallSnapshot"} {+ if !strings.Contains(string(data), want) {+ t.Fatalf("missing %q", want)+ }+ }+ if len(data) > 8*1024*1024+1024 {+ t.Fatal("snapshot exceeds size limit")+ }+ r.record(dir, operationStall{"second", "index", 10 * time.Second})+ again, err := os.ReadFile(path)+ if err != nil || string(data) != string(again) {+ t.Fatal("rate limit did not preserve first snapshot")+ }+ r.last = time.Now().Add(-2 * time.Minute)+ r.record(dir, operationStall{"third", "index", 10 * time.Second})+ again, err = os.ReadFile(path)+ if err != nil || !strings.Contains(string(again), "operation: third") {+ t.Fatal("snapshot was not replaced after rate limit expired")+ }+ files, err := os.ReadDir(dir)+ if err != nil || len(files) != 1 {+ t.Fatal("temporary diagnostic files remain")+ }+}