Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Reduce memory retention in batched editor transactions

Daniel113085c5b7bf
diff --git a/app/src/protyle/util/transactionQueue.test.ts b/app/src/protyle/util/transactionQueue.test.ts
index 136bb107bb..e8be8b9194 100644
--- a/app/src/protyle/util/transactionQueue.test.ts
+++ b/app/src/protyle/util/transactionQueue.test.ts
@@ -107,3 +107,39 @@ test("失败的任务不阻塞后续批次", async () => {
assert.deepEqual(batches, [["next"]]);
});
+
+test("大量积压的更新分批提交并保持任务顺序和完成边界", async () => {
+ const protyle = {} as IProtyle;
+ const blocker = deferred();
+ const batches: number[][] = [];
+ const events: Array<number | string> = [];
+ const promises: Array<Promise<void>> = [];
+ let active = false;
+ const submit = async (items: number[]) => {
+ assert.equal(active, false);
+ active = true;
+ await Promise.resolve();
+ batches.push([...items]);
+ events.push(...items);
+ active = false;
+ };
+ void queueTransaction(protyle, () => blocker.promise);
+ for (let i = 0; i < 70; i++) {
+ promises.push(queueTransactionBatch(protyle, "transactions", i, submit));
+ }
+ void queueTransaction(protyle, async () => {
+ events.push("structure");
+ });
+ void queueTransactionBatch(protyle, "transactions", 70, submit);
+
+ assert.equal(promises[0], promises[31]);
+ assert.notEqual(promises[31], promises[32]);
+ assert.equal(promises[32], promises[63]);
+ assert.notEqual(promises[63], promises[64]);
+ blocker.resolve();
+ await waitForPendingTransactions(protyle);
+ await Promise.all(promises);
+
+ assert.deepEqual(batches.map(items => items.length), [32, 32, 6, 1]);
+ assert.deepEqual(events, [...Array.from({length: 70}, (_, i) => i), "structure", 70]);
+});
diff --git a/app/src/protyle/util/transactionQueue.ts b/app/src/protyle/util/transactionQueue.ts
index d6cf4ba807..7910f9e605 100644
--- a/app/src/protyle/util/transactionQueue.ts
+++ b/app/src/protyle/util/transactionQueue.ts
@@ -10,6 +10,7 @@ interface ITransactionQueue {
}
const transactionQueues = new WeakMap<IProtyle, ITransactionQueue>();
+const MAX_TRANSACTION_BATCH_SIZE = 32;
const getTransactionQueue = (protyle: IProtyle) => {
let queue = transactionQueues.get(protyle);
@@ -43,7 +44,8 @@ export const queueTransactionBatch = <T>(protyle: IProtyle, key: string, item: T
task: (items: T[]) => Promise<void>) => {
const queue = getTransactionQueue(protyle);
const pendingBatch = queue.pendingBatches.get(key) as ITransactionBatch<T> | undefined;
- if (pendingBatch) {
+ // 限制一次请求持有的编辑操作数量,积压的更新按原顺序分批提交。
+ if (pendingBatch && pendingBatch.items.length < MAX_TRANSACTION_BATCH_SIZE) {
pendingBatch.items.push(item);
return pendingBatch.promise;
}
diff --git a/kernel/model/transaction.go b/kernel/model/transaction.go
index a875dc5798..50a039a581 100644
--- a/kernel/model/transaction.go
+++ b/kernel/model/transaction.go
@@ -2591,6 +2591,7 @@ type Transaction struct {
nodes map[string]*ast.Node // 事务中变更的节点
relatedAvIDs []string // 事务中变更的属性视图 ID
changedRootIDs []string // 变更的树 ID 列表(包含了变更定义块后影响的动态锚文本所在的树)
+ mutatedRootIDs []string // 提交后保留实际修改的文档 ID,避免为查询结果持有文档树
boxIcons map[string]string // 事务提交后需要同步的笔记本图标
isGlobalAssetsInit bool // 是否初始化过全局资源判断
@@ -2627,9 +2628,7 @@ type Transaction struct {
}
func (tx *Transaction) GetChangedRootIDs() (ret []string) {
- for t := range tx.trees {
- ret = append(ret, t)
- }
+ ret = tx.GetMutatedRootIDs()
for _, id := range tx.changedRootIDs {
ret = append(ret, id)
@@ -2651,6 +2650,7 @@ func (tx *Transaction) MarkReplay() {
// GetMutatedRootIDs 返回真正被写盘修改结构的树 rootID,不含 refreshDynamicRefTexts 刷新的引用树。
// 用于跨文档撤销判定:单文档编辑返回 1 个 rootID,跨文档移动返回多个,引用文本刷新不计入。
func (tx *Transaction) GetMutatedRootIDs() (ret []string) {
+ ret = append(ret, tx.mutatedRootIDs...)
for t := range tx.trees {
ret = append(ret, t)
}
@@ -2673,6 +2673,8 @@ func (tx *Transaction) begin() (err error) {
tx.templateDocTreeRootSnapshot = nil
tx.trees = map[string]*parse.Tree{}
tx.nodes = map[string]*ast.Node{}
+ tx.mutatedRootIDs = nil
+ tx.changedRootIDs = nil
tx.boxIcons = map[string]string{}
tx.removedCreatedDocs = nil
tx.removedTemplateCreatedDocs = nil
@@ -2883,15 +2885,13 @@ func (tx *Transaction) commit() (err error) {
crossTreeMoveRefRefreshes := append([]crossTreeMoveRefRefresh(nil), tx.crossTreeMoveRefRefreshes...)
IncSync()
+ tx.mutatedRootIDs = tx.GetMutatedRootIDs()
tx.state.Store(2)
committed = true
// 已提交且 trees 稳定后记录到全局撤销日志(rollback 不记录)
GlobalUndoLog.Record(tx)
tx.finishAttributeViewMutation(false)
- tx.blockSwapOriginalTrees = nil
- tx.templateDocTreeRootSnapshot = nil
- tx.attemptedTemplateCreatedDocs = nil
- tx.writeTransactionTree = nil
+ tx.releaseCommittedResources()
tx.m.Unlock()
if 0 < len(crossTreeMoveRefRefreshes) {
task.AppendAsyncTaskWithDelay(task.RefreshCrossTreeMoveRefs, util.SQLFlushInterval,
@@ -2900,6 +2900,24 @@ func (tx *Transaction) commit() (err error) {
return
}
+// releaseCommittedResources 释放执行期间的文档树;操作中的撤销快照及广播所需的文档 ID 继续保留。
+func (tx *Transaction) releaseCommittedResources() {
+ tx.trees, tx.nodes, tx.boxIcons = nil, nil, nil
+ tx.relatedAvIDs = nil
+ tx.removedCreatedDocs, tx.removedTemplateCreatedDocs = nil, nil
+ tx.restoredCreatedDocs, tx.restoredTemplateCreatedDocs = nil, nil
+ tx.restoredCreatedDocBoxes, tx.attemptedTemplateCreatedDocs = nil, nil
+ tx.blockSwapOriginalTrees = nil
+ tx.templateDocTreeRootSnapshot = nil
+ tx.listItemFoldCandidates, tx.listItemFoldCandidateIDs = nil, nil
+ tx.deletedAttrViewBlockIDs, tx.deletedAttrViewCarriers = nil, nil
+ tx.attributeViewDeletionUndo, tx.attributeViewDeletionErr = nil, nil
+ tx.structureCheckNodes = nil
+ tx.crossTreeMoveRefRefreshes = nil
+ tx.removeCreatedDoc, tx.writeTransactionTree = nil, nil
+ tx.luteEngine = nil
+}
+
func (tx *Transaction) rollback() {
tx.finishAttributeViewMutation(true)
for _, tree := range tx.blockSwapOriginalTrees {
diff --git a/kernel/model/transaction_memory_fts5_test.go b/kernel/model/transaction_memory_fts5_test.go
new file mode 100644
index 0000000000..0740ec9b50
--- /dev/null
+++ b/kernel/model/transaction_memory_fts5_test.go
@@ -0,0 +1,111 @@
+//go:build fts5
+
+package model
+
+import (
+ "fmt"
+ "runtime"
+ "slices"
+ "testing"
+ "time"
+ "weak"
+
+ "github.com/88250/lute/ast"
+ "github.com/88250/lute/parse"
+ "github.com/siyuan-note/siyuan/kernel/sql"
+ "github.com/siyuan-note/siyuan/kernel/treenode"
+)
+
+func TestCompletedTransactionBatchReleasesDocumentTrees(t *testing.T) {
+ fixture := setupStructureTransactionTest(t)
+ setupFoldTransactionDatabase(t, fixture)
+ originalUndoLog := GlobalUndoLog
+ GlobalUndoLog = newUndoLog(64)
+ t.Cleanup(func() { GlobalUndoLog = originalUndoLog })
+
+ dom := func(index int) string {
+ return fmt.Sprintf(`<div data-node-id="%s" data-type="NodeParagraph"><div contenteditable="true">edit %d</div></div>`, fixture.childID, index)
+ }
+ var roots []weak.Pointer[ast.Node]
+ transactions := make([]*Transaction, 8)
+ for i := range transactions {
+ tx := &Transaction{
+ DoOperations: []*Operation{{Action: "update", ID: fixture.childID, Data: dom(i)}},
+ UndoOperations: []*Operation{{Action: "update", ID: fixture.childID, Data: dom(i - 1)}},
+ writeTransactionTree: func(tree *parse.Tree) error {
+ roots = append(roots, weak.Make(tree.Root))
+ return writeTreeUpsertQueue(tree)
+ },
+ }
+ tx.MarkFromAPI()
+ transactions[i] = tx
+ }
+ PerformTransactions(&transactions)
+ FlushTxQueue()
+ // 保持整批响应和撤销日志可达,只排空索引队列,验证已完成事务不再持有旧文档树。
+ sql.FlushQueue()
+ if len(roots) != len(transactions) {
+ t.Fatalf("committed %d document trees for %d transactions", len(roots), len(transactions))
+ }
+ // 父文档信息刷新会短暂持有树,等待延迟任务结束后再检查回收。
+ deadline := time.Now().Add(3 * time.Second)
+ for {
+ runtime.GC()
+ retained := -1
+ for index, root := range roots {
+ if root.Value() != nil {
+ retained = index
+ break
+ }
+ }
+ if retained < 0 {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatalf("completed transaction %d still retains its document tree", retained)
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+ for _, tx := range transactions {
+ if tx.state.Load() != 2 || !slices.Equal(tx.GetChangedRootIDs(), []string{fixture.sourceID}) ||
+ !slices.Equal(tx.GetMutatedRootIDs(), []string{fixture.sourceID}) {
+ t.Fatal("releasing document trees changed committed transaction metadata")
+ }
+ }
+ runtime.KeepAlive(transactions)
+
+ assertText := func(want string) {
+ t.Helper()
+ tree, err := LoadTreeByBlockID(fixture.childID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if actual := treenode.GetNodeInTree(tree, fixture.childID).Text(); actual != want {
+ t.Fatalf("persisted block text = %q, want %q", actual, want)
+ }
+ }
+ assertText("edit 7")
+ entry := GlobalUndoLog.Undo(fixture.sourceID)
+ if entry == nil {
+ t.Fatal("committed transaction lost its undo entry")
+ }
+ undo := &Transaction{DoOperations: entry.UndoOperationsForReplay()}
+ undo.MarkReplay()
+ if err := PerformTxSync(undo); err != nil {
+ t.Fatal(err)
+ }
+ GlobalUndoLog.UndoCommit(entry, fixture.sourceID)
+ assertText("edit 6")
+ entry = GlobalUndoLog.Redo(fixture.sourceID)
+ if entry == nil {
+ t.Fatal("undo lost its redo entry")
+ }
+ redo := &Transaction{DoOperations: entry.DoOperationsForReplay()}
+ redo.MarkReplay()
+ if err := PerformTxSync(redo); err != nil {
+ t.Fatal(err)
+ }
+ GlobalUndoLog.RedoCommit(entry, fixture.sourceID)
+ assertText("edit 7")
+ sql.FlushQueue()
+}