Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

:bug: Harden Data.zip import boundaries and recovery https://github.com/siyuan-note/siyuan/issues/20758

Daniel7fcdebb49f0a
diff --git a/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20200924100808-j9sddk9.sy b/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20200924100808-j9sddk9.sy
index 3f45a9cb7f..775df402f1 100644
--- a/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20200924100808-j9sddk9.sy
+++ b/app/guide/20210808180117-6v0mkxr/20200923234011-ieuun1p/20210808180303-xaduj2o/20200924100808-j9sddk9.sy
@@ -6,7 +6,7 @@
"id": "20200924100808-j9sddk9",
"title": "Import and Export",
"type": "doc",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
@@ -416,12 +416,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20261020220502-tsupjxf",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
"Type": "NodeText",
- "Data": "Data.zip can update the current workspace without creating another workspace. The archive is analyzed before importing. If encrypted notebooks conflict, choose Skip conflicts (the default), Overwrite conflicts or Cancel. The result lists skipped notebook IDs before the interface reloads."
+ "Data": "Data.zip can update the current workspace without creating another workspace. The archive is analyzed before importing. If encrypted notebooks conflict, choose Skip conflicts (the default), Overwrite conflicts or Cancel. The result lists skipped notebook IDs before the interface reloads. Cancel is available before submission; the dialog cannot be canceled or closed while the import is being committed."
}
]
},
@@ -444,12 +444,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20261020220502-9jmeu5c",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
"Type": "NodeText",
- "Data": "The import verifies selected content before writing and can roll back a failed or interrupted commit. The original archive, replaced files and key material remain in the workspace’s conf/data-imports/ directory. Keep these recovery files until you have verified the imported content and required history."
+ "Data": "The import verifies selected content before writing and can roll back a failed or interrupted commit. The original archive, replaced files and key material remain in the workspace’s conf/data-imports/ directory. Keep these recovery files until you have verified the imported content and required history. If recovery fails, writes and sync remain paused until recovery succeeds. Unsubmitted preparations expire after 30 minutes and are reclaimed on startup or the next import; expired preparations can still be canceled. Committed recovery files remain available."
}
]
},
diff --git a/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20200813125307-pxsjela.sy b/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20200813125307-pxsjela.sy
index 61e701e75a..76b42d698f 100644
--- a/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20200813125307-pxsjela.sy
+++ b/app/guide/20210808180117-czj9bvb/20200812220555-lj3enxa/20210808180321-hbvl5c2/20200813125307-pxsjela.sy
@@ -6,7 +6,7 @@
"id": "20200813125307-pxsjela",
"title": "导入和导出",
"type": "doc",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
@@ -412,12 +412,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20261020220502-6qiv57v",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
"Type": "NodeText",
- "Data": "Data.zip 可以直接更新当前工作空间,无需新建工作空间。导入前会先分析压缩包,加密笔记本发生冲突时,可选择默认的“跳过冲突”、“覆盖冲突”或取消。导入结果会在界面重载前列出已跳过的笔记本 ID。"
+ "Data": "Data.zip 可以直接更新当前工作空间,无需新建工作空间。导入前会先分析压缩包,加密笔记本发生冲突时,可选择默认的“跳过冲突”、“覆盖冲突”或取消。导入结果会在界面重载前列出已跳过的笔记本 ID。提交前可以取消;提交期间无法取消或关闭导入对话框。"
}
]
},
@@ -440,12 +440,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20261020220502-l5uc6wm",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
"Type": "NodeText",
- "Data": "导入先验证选中内容,再写入数据,失败或中断时可以回滚。上传原件、被覆盖文件和密钥材料保存在工作空间的 conf/data-imports/ 目录;在核对导入内容及所需历史之前,请保留这些恢复文件。"
+ "Data": "导入先验证选中内容,再写入数据,失败或中断时可以回滚。上传原件、被覆盖文件和密钥材料保存在工作空间的 conf/data-imports/ 目录;在核对导入内容及所需历史之前,请保留这些恢复文件。恢复失败时暂停写入和同步,直到恢复成功。未提交的预检在 30 分钟后过期,并在启动或下一次导入时清理,过期后仍可取消;已提交批次的恢复文件继续保留。"
}
]
},
diff --git a/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20211226123302-akitvb1.sy b/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20211226123302-akitvb1.sy
index a4c402f5e9..75e5956fd5 100644
--- a/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20211226123302-akitvb1.sy
+++ b/app/guide/20211226090932-5lcq56f/20211226115423-d5z1joq/20211226121203-rjjngpz/20211226123302-akitvb1.sy
@@ -6,7 +6,7 @@
"id": "20211226123302-akitvb1",
"title": "導入和導出",
"type": "doc",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
@@ -412,12 +412,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20261020220502-4t0tff0",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
"Type": "NodeText",
- "Data": "Data.zip 可以直接更新目前的工作空間,無須建立另一個工作空間。匯入前會先分析壓縮檔,加密筆記本發生衝突時,可選擇預設的「略過衝突」、「覆寫衝突」或取消。匯入結果會在介面重新載入前列出已略過的筆記本 ID。"
+ "Data": "Data.zip 可以直接更新目前的工作空間,無須建立另一個工作空間。匯入前會先分析壓縮檔,加密筆記本發生衝突時,可選擇預設的「略過衝突」、「覆寫衝突」或取消。匯入結果會在介面重新載入前列出已略過的筆記本 ID。提交前可以取消;提交期間無法取消或關閉匯入對話框。"
}
]
},
@@ -440,12 +440,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20261020220502-s7xahw4",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
"Type": "NodeText",
- "Data": "匯入先驗證選取內容,再寫入資料,失敗或中斷時可以回復。上傳原件、被覆寫檔案及金鑰材料保存在工作空間的 conf/data-imports/ 目錄;核對匯入內容及所需歷史之前,請保留這些復原檔案。"
+ "Data": "匯入先驗證選取內容,再寫入資料,失敗或中斷時可以回復。上傳原件、被覆寫檔案及金鑰材料保存在工作空間的 conf/data-imports/ 目錄;核對匯入內容及所需歷史之前,請保留這些復原檔案。復原失敗時暫停寫入和同步,直到復原成功。尚未提交的預檢在 30 分鐘後過期,並在啟動或下一次匯入時清理,過期後仍可取消;已提交批次的復原檔案會繼續保留。"
}
]
},
diff --git a/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-d31i9v5.sy b/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-d31i9v5.sy
index bb4c43f4fb..c995070a12 100644
--- a/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-d31i9v5.sy
+++ b/app/guide/20240530133126-axarxgx/20240530101000-4qitucx/20240530101000-g3ugxml/20240530101000-d31i9v5.sy
@@ -7,7 +7,7 @@
"id": "20240530101000-d31i9v5",
"title": "インポートとエクスポート",
"type": "doc",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
@@ -423,12 +423,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20261020220502-x9fsafc",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
"Type": "NodeText",
- "Data": "Data.zip は別のワークスペースを作成せずに、現在のワークスペースを更新できます。インポート前にアーカイブを解析し、暗号化ノートブックが競合する場合は、初期選択の「競合をスキップ」、「競合を上書き」、またはキャンセルを選べます。画面の再読み込み前に、スキップしたノートブックの ID が結果に表示されます。"
+ "Data": "Data.zip は別のワークスペースを作成せずに、現在のワークスペースを更新できます。インポート前にアーカイブを解析し、暗号化ノートブックが競合する場合は、初期選択の「競合をスキップ」、「競合を上書き」、またはキャンセルを選べます。画面の再読み込み前に、スキップしたノートブックの ID が結果に表示されます。実行前はキャンセルできます。処理中はキャンセルやダイアログの終了はできません。"
}
]
},
@@ -451,12 +451,12 @@
"Type": "NodeParagraph",
"Properties": {
"id": "20261020220502-c9dn1ha",
- "updated": "20261020220502"
+ "updated": "20261020225315"
},
"Children": [
{
"Type": "NodeText",
- "Data": "選択した内容を検証してからデータを書き込み、失敗や中断が発生した場合はロールバックできます。元のアーカイブ、上書き前のファイル、キー情報はワークスペースの conf/data-imports/ ディレクトリに保持されます。取り込まれた内容と必要な履歴を確認するまでは、これらの復旧ファイルを保持してください。"
+ "Data": "選択した内容を検証してからデータを書き込み、失敗や中断が発生した場合はロールバックできます。元のアーカイブ、上書き前のファイル、キー情報はワークスペースの conf/data-imports/ ディレクトリに保持されます。取り込まれた内容と必要な履歴を確認するまでは、これらの復旧ファイルを保持してください。復旧に失敗した場合は、復旧が成功するまで書き込みと同期を停止します。未実行の事前解析は 30 分で期限切れになり、起動時または次のインポート時に削除されます。期限切れ後もキャンセルでき、実行済みの復旧ファイルは保持されます。"
}
]
},
diff --git a/app/src/dialog/beforeClose.test.ts b/app/src/dialog/beforeClose.test.ts
new file mode 100644
index 0000000000..1fee4cd31b
--- /dev/null
+++ b/app/src/dialog/beforeClose.test.ts
@@ -0,0 +1,43 @@
+import {strict as assert} from "node:assert";
+import {readFileSync} from "node:fs";
+import {test} from "node:test";
+import {runInNewContext} from "node:vm";
+import {ModuleKind, ScriptTarget, transpileModule} from "typescript";
+
+test("dialog close guard covers direct close requests and permits cleanup after completion", () => {
+ const compiled = transpileModule(readFileSync("src/dialog/index.ts", "utf8"), {
+ compilerOptions: {module: ModuleKind.CommonJS, target: ScriptTarget.ES2021},
+ }).outputText;
+ const exported = {} as {Dialog: {prototype: object}};
+ const scheduled: Array<() => void> = [];
+ let panelsDestroyed = 0;
+ runInNewContext(compiled, {
+ exports: exported,
+ require: () => ({
+ isAbove: () => false,
+ Constants: {TIMEOUT_DBLCLICK: 0},
+ destroyDialogBlockPanels: () => panelsDestroyed++,
+ }),
+ document: {removeEventListener: () => {}},
+ window: {siyuan: {menus: {menu: {element: {}}}}},
+ setTimeout: (callback: () => void) => scheduled.push(callback),
+ });
+ let allowClose = false;
+ const dialog = Object.assign(Object.create(exported.Dialog.prototype), {
+ destroying: false,
+ closeGuard: () => allowClose,
+ element: {dataset: {}, classList: {remove: () => {}}, querySelector: () => ({})},
+ });
+ dialog.destroy();
+ dialog.destroy();
+ assert.equal(dialog.destroying, false);
+ assert.equal(scheduled.length, 0);
+ assert.equal(panelsDestroyed, 0);
+ allowClose = true;
+ dialog.destroy();
+ assert.equal(dialog.destroying, true);
+ assert.equal(scheduled.length, 1);
+ assert.equal(panelsDestroyed, 1);
+ dialog.destroy();
+ assert.equal(scheduled.length, 1);
+});
diff --git a/app/src/dialog/index.ts b/app/src/dialog/index.ts
index 7c647f7eb9..96b7155775 100644
--- a/app/src/dialog/index.ts
+++ b/app/src/dialog/index.ts
@@ -13,6 +13,7 @@ export class Dialog {
public element: HTMLElement;
private id: string;
private disableClose: boolean;
+ private closeGuard: () => boolean;
public editors: { [key: string]: Protyle };
public data: any;
private resizeCallback: (type: string) => void;
@@ -65,6 +66,8 @@ export class Dialog {
width?: string,
height?: string,
destroyCallback?: (options?: IObject) => void,
+ // 返回 false 时保留对话框,覆盖按钮、键盘、移动端返回和直接销毁调用。
+ beforeClose?: () => boolean,
disableClose?: boolean,
hideCloseIcon?: boolean,
disableAnimation?: boolean,
@@ -82,6 +85,7 @@ export class Dialog {
}
this.resizeCallback = options.resizeCallback;
this.disableClose = options.disableClose;
+ this.closeGuard = options.beforeClose;
this.id = genUUID();
window.siyuan.dialogs.push(this);
this.destroyCallback = options.destroyCallback;
@@ -149,7 +153,7 @@ left:${left || "auto"};top:${top || "auto"}">
}
public destroy(options?: IObject) {
- if (this.destroying) {
+ if (this.destroying || this.closeGuard?.() === false) {
return;
}
this.destroying = true;
diff --git a/app/src/menus/importData.test.ts b/app/src/menus/importData.test.ts
index 1bbced2d0a..8be559a82e 100644
--- a/app/src/menus/importData.test.ts
+++ b/app/src/menus/importData.test.ts
@@ -23,7 +23,7 @@ class Control {
}
}
-const setup = (encrypted = ["box"], conflicts = ["box"]) => {
+const setup = (encrypted = ["box"], conflicts = ["box"], commitWait = Promise.resolve()) => {
const requests: Array<{url: string, data: {id?: string, mode?: string, password?: string}}> = [];
const dialogs: MockDialog[] = [];
let selectedMode = "skip";
@@ -45,14 +45,14 @@ const setup = (encrypted = ["box"], conflicts = ["box"]) => {
},
querySelectorAll: () => this.modes,
};
- constructor(public options: {content: string, width: string, destroyCallback: () => void}) {
+ constructor(public options: {content: string, width: string, destroyCallback: () => void, beforeClose?: () => boolean}) {
for (const name of ["password", "password-label", "import", "cancel", "confirm"]) {
this.controls.set(`[data-type="${name}"]`, new Control());
}
dialogs.push(this);
}
destroy() {
- if (!this.destroyed) {
+ if (!this.destroyed && this.options.beforeClose?.() !== false) {
this.destroyed = true;
this.options.destroyCallback();
}
@@ -77,6 +77,7 @@ const setup = (encrypted = ["box"], conflicts = ["box"]) => {
return {code: prepareCode, data: {id: "plan-id", encrypted, conflicts}};
}
if (url.endsWith("commitDataImport")) {
+ await commitWait;
return {code: commitCode, data: {skipped: data.mode === "skip" ? conflicts : []}};
}
return {code: 0};
@@ -147,6 +148,24 @@ test("Data.zip cancellation only cancels the prepared upload", async () => {
assert.equal(state.requests.some((item) => item.url.endsWith("commitDataImport")), false);
});
+test("Data.zip blocks cancel and shared dialog close paths while commit is pending", async () => {
+ let finish: () => void;
+ const wait = new Promise<void>((resolve) => finish = resolve);
+ const state = setup([], [], wait);
+ await state.exports.importDataFile({} as File);
+ const dialog = state.dialogs[0];
+ const committing = dialog.control("import").dispatch("click");
+ assert.equal(dialog.control("cancel").disabled, true);
+ await dialog.control("cancel").dispatch("click");
+ dialog.destroy();
+ assert.equal(dialog.destroyed, false);
+ assert.equal(state.requests.some((item) => item.url.endsWith("cancelDataImport")), false);
+ finish();
+ await committing;
+ assert.equal(dialog.destroyed, true);
+ assert.equal(state.dialogs.length, 2);
+});
+
test("Data.zip still authenticates non-conflicting encrypted notebooks when conflicts are skipped", () => {
const state = setup();
assert.equal(state.exports.needsDataImportPassword({encrypted: ["old", "new"], conflicts: ["old"]}, "skip"), true);
diff --git a/app/src/menus/importData.ts b/app/src/menus/importData.ts
index a729545167..4e57a27d6c 100644
--- a/app/src/menus/importData.ts
+++ b/app/src/menus/importData.ts
@@ -30,6 +30,8 @@ export const importDataFile = async (file: File, onComplete?: () => void) => {
};
const dialog = new Dialog({
title: `${lang.import} Data`,
+ hideCloseIcon: true,
+ beforeClose: () => !busy || committed,
width: isMobile() ? "92vw" : "560px",
content: `<div class="b3-dialog__content">
<div class="ft__breakword">${lang.importDataTip}</div>
@@ -55,6 +57,7 @@ export const importDataFile = async (file: File, onComplete?: () => void) => {
});
const password = dialog.element.querySelector<HTMLInputElement>('[data-type="password"]');
const button = dialog.element.querySelector<HTMLButtonElement>('[data-type="import"]');
+ const cancelButton = dialog.element.querySelector<HTMLButtonElement>('[data-type="cancel"]');
const modes = dialog.element.querySelectorAll<HTMLInputElement>('[name="data-import-mode"]');
const mode = (): "skip" | "overwrite" =>
dialog.element.querySelector<HTMLInputElement>('[name="data-import-mode"]:checked')?.value === "overwrite" ? "overwrite" : "skip";
@@ -63,11 +66,16 @@ export const importDataFile = async (file: File, onComplete?: () => void) => {
dialog.element.querySelector('[data-type="password-label"]').classList.toggle("fn__none", !required);
password.required = required;
password.disabled = busy;
+ cancelButton.disabled = busy;
button.disabled = busy || required && !password.value.trim();
};
modes.forEach((item) => item.addEventListener("change", update));
password.addEventListener("input", update);
- dialog.element.querySelector('[data-type="cancel"]').addEventListener("click", () => dialog.destroy());
+ cancelButton.addEventListener("click", () => {
+ if (!busy) {
+ dialog.destroy();
+ }
+ });
button.addEventListener("click", async () => {
if (busy || button.disabled) {
return;
diff --git a/app/src/types/api/index.d.ts b/app/src/types/api/index.d.ts
index 988a97b4cf..549056e329 100644
--- a/app/src/types/api/index.d.ts
+++ b/app/src/types/api/index.d.ts
@@ -5677,7 +5677,7 @@ export interface APIPOSTRoutes {
authorization?: readonly ["authenticated","admin"];
};
/**
- * 取消尚未提交且仍有效的 Data.zip 预检;不能删除已提交批次的恢复材料。
+ * 取消尚未提交的 Data.zip 预检,包括过期预检;不能删除已提交批次的恢复材料。
*/
"/api/import/cancelDataImport": {
request: DataImportRequestInput;
@@ -5796,6 +5796,8 @@ export interface APIPOSTRoutes {
/**
* 预检 Data.zip 并保存原件,返回加密冲突及需要认证的笔记本 ID,不修改活动数据。
* 预检有效期为 30 分钟;提交前必须显式选择 skip 或 overwrite,取消会移除尚未提交的暂存文件。
+ * 过期且未提交的预检在启动或下一次导入时清理;预检失败时也清理本次暂存副本。
+ * 存在未完成批次时,先在独占访问下恢复;恢复失败会继续阻断其他写入、笔记本操作和同步。
*/
"/api/import/prepareDataImport": {
request: ImportDataRequestInput;
diff --git a/docs/ENCRYPTED-NOTEBOOK.md b/docs/ENCRYPTED-NOTEBOOK.md
index 6e67096368..a1be2ae007 100644
--- a/docs/ENCRYPTED-NOTEBOOK.md
+++ b/docs/ENCRYPTED-NOTEBOOK.md
@@ -186,10 +186,14 @@ Sync recovery can validate backup integrity but cannot prove that a backup is th
Data.zip imports can update the current workspace on desktop and mobile. The import is analyzed before any active data is written. When an encrypted notebook conflicts with an existing notebook, the dialog defaults to skipping conflicts and also offers overwrite and cancel. Skipping preserves the notebook directory and its key material; the result lists skipped notebook IDs before the interface reloads. Overwrite retains the existing path-based semantics: matching paths are replaced and destination-only files are preserved. It is not a mirror restore and does not propagate deletions.
+Noncanonical notebook path aliases are rejected before import. Cancel is available before submission; while committing, the dialog cannot be canceled or closed, including with Esc or the mobile back action. Unsubmitted preparations expire after 30 minutes and are reclaimed on startup or the next import; expired preparations can still be canceled. Committed recovery files remain available.
+
Importing encrypted content requires a master password that authenticates a compatible key configuration, notebook envelopes, metadata, documents, assets and database definitions. Backups from before a password change can use authenticated historical keys without reverting the current password or discarding recovery keys. The same notebook ID or password string alone does not establish compatibility. Different encryption states or different data keys cannot be merged directly. An incomplete local key configuration must be recovered before importing encrypted content. The source archive's password-migration journal is never installed as an active operation. Existing ciphertext formats, AAD and key derivation remain unchanged.
Affected notebooks stop accepting new operations and wait for existing operations before replacement; they remain locked after import. All selected content is staged and verified before commit. Replaced files and key material are retained with the uploaded archive under `<workspace>/conf/data-imports/<operation ID>/`; a persistent journal restores unfinished batches before notebook mounting or sync. Retain these recovery files until the imported data and required history have been verified. Legacy `/api/import/importData`, CLI and MCP callers do not silently overwrite encrypted conflicts; the interactive workflow uses the typed prepare, commit and cancel endpoints.
+If rollback itself fails, writes, new notebook operations, key adoption and sync remain blocked. Starting another import retries recovery under exclusive access before preparing the archive; admission resumes only after recovery succeeds. Recovery also runs before notebook mounting and sync at startup.
+
#### What if I forget the password
It cannot be recovered — by design (no backdoor). Even if the ciphertext has been synced to the cloud, it cannot be decrypted without the master password. You must remember the master password; using a password manager is recommended.
diff --git a/docs/ENCRYPTED-NOTEBOOK.zh-CN.md b/docs/ENCRYPTED-NOTEBOOK.zh-CN.md
index 50421eb710..39a4b73022 100644
--- a/docs/ENCRYPTED-NOTEBOOK.zh-CN.md
+++ b/docs/ENCRYPTED-NOTEBOOK.zh-CN.md
@@ -186,10 +186,14 @@ Windows 和 macOS 桌面端在 设置 - 鉴权 - 加密笔记本 中提供「跟
桌面和移动端可以通过导入 Data.zip 更新当前工作空间。导入先进行分析,再写入活动数据。加密笔记本与现有笔记本冲突时,对话框默认跳过冲突,并提供覆盖和取消。跳过会保留笔记本目录及其密钥材料,结果在界面重载前列出跳过的笔记本 ID。覆盖沿用按路径覆盖语义:相同路径替换,目标独有的文件保留;它不是镜像恢复,也不会传播删除。
+导入前会拒绝非规范的笔记本路径别名。提交前可以取消;提交期间无法取消或关闭对话框,Esc 和移动端返回操作也不能关闭。未提交的预检在 30 分钟后过期,并在启动或下一次导入时清理,过期后仍可取消;已提交批次的恢复文件继续保留。
+
导入加密内容需要输入主密码,认证兼容的密钥配置、笔记本包络、元数据、文档、资源和数据库定义。改密前的备份可以使用经过认证的历史密钥读取,不回退当前密码或丢弃恢复密钥。仅凭笔记本 ID 或密码字符串相同不能判定兼容,加密状态不同或数据密钥不同的笔记本不能直接合并。本机密钥配置不完整时,应先完成恢复,再导入加密内容。来源压缩包的改密事务记录不会作为活动操作安装。已有密文格式、AAD 和密钥派生保持不变。
受影响的笔记本先停止接纳新操作并等待在途操作结束,再执行替换,导入后保持锁定。全部选中内容在提交前完成暂存和验证。上传原件、被覆盖文件和密钥材料保存在 `<工作空间>/conf/data-imports/<操作 ID>/`,持久化日志在挂载和同步前恢复未完成批次。在验证导入数据及所需历史之前应保留这些恢复文件。旧 `/api/import/importData`、CLI 和 MCP 调用不会静默覆盖加密冲突;交互流程使用有类型契约的预检、提交和取消接口。
+如果回滚本身失败,写入、新的笔记本操作、密钥配置采用和同步会保持阻断。再次发起导入时,先在独占访问下重试恢复,再预检压缩包;恢复成功后才重新放行。启动时也会在挂载笔记本和同步之前执行恢复。
+
#### 忘记密码怎么办
无法恢复——这是设计使然(没有后门)。即使密文已同步到云端,没有主密码也解不开。务必牢记主密码,建议使用密码管理器保存。
diff --git a/kernel/api/contract_import_data_test.go b/kernel/api/contract_import_data_test.go
index dfa301e468..512eee77c9 100644
--- a/kernel/api/contract_import_data_test.go
+++ b/kernel/api/contract_import_data_test.go
@@ -27,6 +27,7 @@ import (
"path/filepath"
"strings"
"testing"
+ "time"
"github.com/gin-gonic/gin"
"github.com/siyuan-note/siyuan/kernel/apicontract"
@@ -50,9 +51,10 @@ func TestAPIContractDataImportPrepareCancelAndInvalidCommit(t *testing.T) {
model.Conf, util.DataDir, util.ConfDir, util.TempDir, util.HistoryDir = oldConf, oldData, oldConfig, oldTemp, oldHistory
})
engine := gin.New()
- engine.POST("/api/import/prepareDataImport", prepareDataImport)
- engine.POST("/api/import/commitDataImport", commitDataImport)
- engine.POST("/api/import/cancelDataImport", cancelDataImport)
+ engine.Use(func(c *gin.Context) { c.Set(model.RoleContextKey, model.RoleAdministrator) })
+ engine.POST("/api/import/prepareDataImport", model.CheckReadonly, prepareDataImport)
+ engine.POST("/api/import/commitDataImport", model.CheckReadonly, commitDataImport)
+ engine.POST("/api/import/cancelDataImport", model.CheckReadonly, cancelDataImport)
var archive bytes.Buffer
writer := archivezip.NewWriter(&archive)
file, err := writer.Create("data/ordinary.txt")
@@ -108,6 +110,23 @@ func TestAPIContractDataImportPrepareCancelAndInvalidCommit(t *testing.T) {
t.Fatalf("accepted invalid commit: %s", recorder.Body.String())
}
}
+ stagePath := filepath.Join(util.ConfDir, "data-imports", prepared.Data.ID, "stage.json")
+ stageData, err := os.ReadFile(stagePath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var stage map[string]any
+ if err = json.Unmarshal(stageData, &stage); err != nil {
+ t.Fatal(err)
+ }
+ stage["created"] = time.Now().Add(-time.Hour).Unix()
+ stageData, err = json.Marshal(stage)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err = os.WriteFile(stagePath, stageData, 0600); err != nil {
+ t.Fatal(err)
+ }
recorder = httptest.NewRecorder()
request = httptest.NewRequest(http.MethodPost, "/api/import/cancelDataImport", strings.NewReader(`{"id":" `+prepared.Data.ID+` "}`))
engine.ServeHTTP(recorder, request)
diff --git a/kernel/apicontract/contracts.go b/kernel/apicontract/contracts.go
index f087d870f5..eb1fc15d89 100644
--- a/kernel/apicontract/contracts.go
+++ b/kernel/apicontract/contracts.go
@@ -682,6 +682,8 @@ var ImportData = define[ImportDataRequest, Null]("importData", "/api/import/impo
// 预检 Data.zip 并保存原件,返回加密冲突及需要认证的笔记本 ID,不修改活动数据。
// 预检有效期为 30 分钟;提交前必须显式选择 skip 或 overwrite,取消会移除尚未提交的暂存文件。
+// 过期且未提交的预检在启动或下一次导入时清理;预检失败时也清理本次暂存副本。
+// 存在未完成批次时,先在独占访问下恢复;恢复失败会继续阻断其他写入、笔记本操作和同步。
var PrepareDataImport = define[ImportDataRequest, DataImportPlan]("prepareDataImport", "/api/import/prepareDataImport", AuthenticatedAccess|AdminAccess|WritableAccess, MultipartBody, ResponseOptions{}, "POST")
// 按路径覆盖并保留目标独有文件,skip 保留冲突目录及其密钥材料,返回明确的跳过清单。
@@ -690,7 +692,7 @@ var PrepareDataImport = define[ImportDataRequest, DataImportPlan]("prepareDataIm
// 成功批次的原件、旧密文和密钥材料保留在工作空间 conf/data-imports 下。
var CommitDataImport = define[CommitDataImportRequest, DataImportResult]("commitDataImport", "/api/import/commitDataImport", AuthenticatedAccess|AdminAccess|WritableAccess, JSONBody, ResponseOptions{}, "POST")
-// 取消尚未提交且仍有效的 Data.zip 预检;不能删除已提交批次的恢复材料。
+// 取消尚未提交的 Data.zip 预检,包括过期预检;不能删除已提交批次的恢复材料。
var CancelDataImport = define[DataImportRequest, Null]("cancelDataImport", "/api/import/cancelDataImport", AuthenticatedAccess|AdminAccess|WritableAccess, JSONBody, ResponseOptions{}, "POST")
// 压缩包中的 Markdown 使用与 `/api/import/importStdMd` 相同的标准脚注转换规则。
diff --git a/kernel/model/crypto.go b/kernel/model/crypto.go
index be47f23fa7..d33ab93faf 100644
--- a/kernel/model/crypto.go
+++ b/kernel/model/crypto.go
@@ -1237,6 +1237,9 @@ func deriveNotebookCryptoBackupCandidate(password string) (backup *conf.Notebook
// deriveKEK 从主密码派生 KEK 并校验。校验失败返回错误。KEK 仅在函数作用域内有效,调用方负责使用。
func deriveKEK(password string) ([]byte, error) {
+ if err := checkDataImportRecovery(); err != nil {
+ return nil, err
+ }
if err := checkPendingNotebookArchives(); err != nil {
return nil, err
}
diff --git a/kernel/model/crypto_lifecycle.go b/kernel/model/crypto_lifecycle.go
index 2e3312df94..4ce01f4d14 100644
--- a/kernel/model/crypto_lifecycle.go
+++ b/kernel/model/crypto_lifecycle.go
@@ -149,6 +149,9 @@ func notebookCryptoConfigurationComplete(notebookCrypto *conf.NotebookCrypto) bo
// AcquireEncryptedBoxOperation 为需要把明文结果保留到响应结束的操作取得准入租约。
func AcquireEncryptedBoxOperation(boxID string) error {
+ if err := checkDataImportRecovery(); err != nil {
+ return err
+ }
if boxID == "" || !IsEncryptedBox(boxID) {
return nil
}
diff --git a/kernel/model/import_data.go b/kernel/model/import_data.go
index 6c68014bd9..63ed16ca5a 100644
--- a/kernel/model/import_data.go
+++ b/kernel/model/import_data.go
@@ -30,6 +30,7 @@ import (
"sort"
"strings"
"sync"
+ "sync/atomic"
"time"
"github.com/88250/lute/ast"
@@ -41,6 +42,14 @@ const dataImportTTL = 30 * time.Minute
var dataImportMu sync.Mutex
var dataImportAccess sync.RWMutex
+var dataImportRecoveryRequired atomic.Bool
+
+func checkDataImportRecovery() error {
+ if dataImportRecoveryRequired.Load() {
+ return errors.New(Conf.Language(424))
+ }
+ return nil
+}
type DataImportPlan struct {
ID string `json:"id"`
@@ -68,11 +77,14 @@ func dataImportDirectory(id string) (string, error) {
return directory, nil
}
-// PrepareDataImport 仅保存上传原件和预检结果,不修改活动数据或解锁笔记本。
+// PrepareDataImport 先在独占访问下恢复未完成批次,再保存上传原件和预检结果,不安装上传内容。
func PrepareDataImport(zipPath string) (plan DataImportPlan, err error) {
+ if err = recoverDataImportsForPrepare(); err != nil {
+ return
+ }
dataImportMu.Lock()
defer dataImportMu.Unlock()
- if err = recoverDataImports(); err != nil {
+ if err = checkDataImportRecovery(); err != nil {
return
}
plan = DataImportPlan{ID: ast.NewNodeID(), Conflicts: []string{}, Encrypted: []string{}}
@@ -80,6 +92,11 @@ func PrepareDataImport(zipPath string) (plan DataImportPlan, err error) {
if err != nil {
return
}
+ defer func() {
+ if err != nil {
+ err = errors.Join(err, os.RemoveAll(directory))
+ }
+ }()
if err = archiveCopyFile(zipPath, filepath.Join(directory, "source.zip")); err != nil {
return
}
@@ -176,6 +193,9 @@ func inspectDataImportNotebooks(root string) (map[string]*conf.BoxEncryption, er
return nil, err
}
identities := map[string]*conf.BoxEncryption{}
+ if err = validateDataImportNotebookPaths(root, entries); err != nil {
+ return nil, err
+ }
for _, entry := range entries {
if strings.HasSuffix(entry.Name(), ".sy") {
return nil, errors.New(Conf.Language(198))
@@ -236,15 +256,66 @@ func inspectDataImportNotebooks(root string) (map[string]*conf.BoxEncryption, er
return identities, nil
}
+// validateDataImportNotebookPaths 拒绝非规范 ID,并逐级核对已有目录项,防止大小写和短文件名别名绕过身份检查。
+func validateDataImportNotebookPaths(root string, entries []os.DirEntry) error {
+ for _, entry := range entries {
+ if name := entry.Name(); ast.IsNodeIDPattern(strings.ToLower(name)) && !ast.IsNodeIDPattern(name) {
+ return errors.New(Conf.Language(198))
+ }
+ }
+ directories := map[string]map[string]bool{}
+ return archiveWalk(root, func(name, _ string) error {
+ parent := util.DataDir
+ for _, part := range strings.Split(name, "/") {
+ names, cached := directories[parent]
+ if !cached {
+ children, err := os.ReadDir(parent)
+ if os.IsNotExist(err) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ names = map[string]bool{}
+ for _, child := range children {
+ names[child.Name()] = true
+ }
+ directories[parent] = names
+ }
+ parent = filepath.Join(parent, part)
+ if names[part] {
+ continue
+ }
+ if _, err := os.Lstat(parent); err == nil {
+ return errors.New(Conf.Language(198))
+ } else if !os.IsNotExist(err) {
+ return err
+ }
+ return nil
+ }
+ return nil
+ })
+}
+
func loadDataImportStage(id string) (string, *dataImportStage, error) {
+ directory, stage, err := readDataImportStage(id)
+ if err != nil {
+ return "", nil, err
+ }
+ if time.Now().Unix()-stage.Created > int64(dataImportTTL.Seconds()) || stage.Created > time.Now().Unix() {
+ return "", nil, errors.New(Conf.Language(422))
+ }
+ return directory, stage, nil
+}
+
+func readDataImportStage(id string) (string, *dataImportStage, error) {
directory, err := dataImportDirectory(id)
if err != nil {
return "", nil, err
}
data, err := os.ReadFile(filepath.Join(directory, "stage.json"))
stage := &dataImportStage{}
- if err != nil || json.Unmarshal(data, stage) != nil || stage.Version != 1 || stage.Plan.ID != id ||
- time.Now().Unix()-stage.Created > int64(dataImportTTL.Seconds()) || stage.Created > time.Now().Unix() {
+ if err != nil || json.Unmarshal(data, stage) != nil || stage.Version != 1 || stage.Plan.ID != id || stage.Created <= 0 {
return "", nil, errors.New(Conf.Language(422))
}
if _, err = os.Lstat(filepath.Join(directory, "operation.json")); !os.IsNotExist(err) {
@@ -256,7 +327,7 @@ func loadDataImportStage(id string) (string, *dataImportStage, error) {
func CancelDataImport(id string) error {
dataImportMu.Lock()
defer dataImportMu.Unlock()
- directory, _, err := loadDataImportStage(id)
+ directory, _, err := readDataImportStage(id)
if err != nil {
return err
}
diff --git a/kernel/model/import_data_test.go b/kernel/model/import_data_test.go
index 1ca396609e..b3f842acb8 100644
--- a/kernel/model/import_data_test.go
+++ b/kernel/model/import_data_test.go
@@ -27,6 +27,7 @@ import (
"path/filepath"
"reflect"
"strings"
+ "sync"
"testing"
"time"
@@ -190,6 +191,58 @@ func TestDataImportCanSkipEveryFile(t *testing.T) {
}
}
+func TestDataImportRejectsNotebookPathAliases(t *testing.T) {
+ kek := dataImportTestWorkspace(t)
+ defer zeroAndClear(kek)
+ source := t.TempDir()
+ alias := strings.ToUpper(dataImportTestID)
+ if err := writeDataImportJSON(filepath.Join(source, alias, ".siyuan", "conf.json"), conf.NewBoxConf()); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(source, alias, dataImportTestID+".sy"), []byte("unauthenticated replacement"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ before, err := archiveDirectoryFiles(util.DataDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = PrepareDataImport(dataImportTestZip(t, source)); err == nil {
+ t.Fatal("accepted a noncanonical notebook ID that aliases an encrypted directory")
+ }
+ if _, err = validateImportedNotebookIdentities(source); err == nil {
+ t.Fatal("legacy import accepted a notebook path alias")
+ }
+ after, err := archiveDirectoryFiles(util.DataDir)
+ if err != nil || !reflect.DeepEqual(before, after) {
+ t.Fatalf("path alias changed active data: %v", err)
+ }
+}
+
+func TestDataImportRejectsNestedDestinationAliases(t *testing.T) {
+ kek := dataImportTestWorkspace(t)
+ defer zeroAndClear(kek)
+ if _, err := os.Stat(filepath.Join(util.DataDir, ".SIYUAN")); os.IsNotExist(err) {
+ t.Skip("filesystem distinguishes directory name case")
+ } else if err != nil {
+ t.Fatal(err)
+ }
+ source := t.TempDir()
+ if err := writeDataImportBytes(filepath.Join(source, ".SIYUAN", "unrecognized-key.json"), []byte("source")); err != nil {
+ t.Fatal(err)
+ }
+ before, err := archiveDirectoryFiles(util.DataDir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = PrepareDataImport(dataImportTestZip(t, source)); err == nil {
+ t.Fatal("accepted a destination directory alias outside notebook roots")
+ }
+ after, err := archiveDirectoryFiles(util.DataDir)
+ if err != nil || !reflect.DeepEqual(before, after) {
+ t.Fatalf("directory alias changed active data: %v", err)
+ }
+}
+
func TestDataImportRestoresEncryptedNotebookInUnconfiguredWorkspace(t *testing.T) {
kek := dataImportTestWorkspace(t)
defer zeroAndClear(kek)
@@ -329,6 +382,12 @@ func TestDataImportRejectsUnsafeArchives(t *testing.T) {
if _, err = PrepareDataImport(p); err == nil {
t.Fatal("accepted an unsafe archive")
}
+ if entries, err := os.ReadDir(dataImportBase()); err != nil || len(entries) != 0 {
+ t.Fatalf("failed preflight retained temporary uploads: %v", err)
+ }
+ if _, err := os.Stat(p); err != nil {
+ t.Fatalf("failed preflight removed the original archive: %v", err)
+ }
files, err := archiveDirectoryFiles(util.DataDir)
if err != nil || len(files) != 0 {
t.Fatalf("invalid archive changed active data: %v", err)
@@ -424,6 +483,51 @@ func TestDataImportBlocksOtherWriteRequestsDuringCommit(t *testing.T) {
}
}
+func TestDataImportPreparationWaitsForExistingWriteRequests(t *testing.T) {
+ notebookArchiveTestWorkspace(t)
+ previousReadonly := util.ReadOnly
+ util.ReadOnly = false
+ t.Cleanup(func() { util.ReadOnly = previousReadonly })
+ source := t.TempDir()
+ if err := os.WriteFile(filepath.Join(source, "ordinary.txt"), []byte("source"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ archive := dataImportTestZip(t, source)
+ entered, release, finished := make(chan struct{}), make(chan struct{}), make(chan struct{})
+ var once sync.Once
+ releaseWriter := func() { once.Do(func() { close(release) }) }
+ defer releaseWriter()
+ engine := gin.New()
+ engine.Use(func(c *gin.Context) { c.Set(RoleContextKey, RoleAdministrator) })
+ engine.POST("/api/test/write", CheckReadonly, func(c *gin.Context) {
+ close(entered)
+ <-release
+ c.Status(http.StatusNoContent)
+ })
+ go func() {
+ engine.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/api/test/write", nil))
+ close(finished)
+ }()
+ <-entered
+ prepared := make(chan error, 1)
+ go func() { _, err := PrepareDataImport(archive); prepared <- err }()
+ select {
+ case err := <-prepared:
+ t.Fatalf("preparation recovery entered while another writer was active: %v", err)
+ case <-time.After(30 * time.Millisecond):
+ }
+ releaseWriter()
+ select {
+ case err := <-prepared:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(5 * time.Second):
+ t.Fatal("preparation remained blocked after the writer finished")
+ }
+ <-finished
+}
+
func TestDataImportAdoptsNewerCompatibleKeyConfiguration(t *testing.T) {
kek := dataImportTestWorkspace(t)
defer zeroAndClear(kek)
@@ -569,6 +673,161 @@ func TestDataImportCancelChangesAndExpiration(t *testing.T) {
if _, err = CommitDataImport(plan.ID, "overwrite", ""); err == nil {
t.Fatal("accepted expired preflight")
}
+ if _, err = os.Stat(directory); !os.IsNotExist(err) {
+ t.Fatal("expired preparation was not reclaimed")
+ }
+}
+
+func TestDataImportExpiredCancelAndCleanup(t *testing.T) {
+ notebookArchiveTestWorkspace(t)
+ source := t.TempDir()
+ if err := os.WriteFile(filepath.Join(source, "ordinary.txt"), []byte("source"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ prepareExpired := func() (DataImportPlan, string) {
+ plan, err := PrepareDataImport(dataImportTestZip(t, source))
+ if err != nil {
+ t.Fatal(err)
+ }
+ directory, stage, err := loadDataImportStage(plan.ID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ stage.Created = time.Now().Add(-time.Hour).Unix()
+ if err = writeDataImportJSON(filepath.Join(directory, "stage.json"), stage); err != nil {
+ t.Fatal(err)
+ }
+ return plan, directory
+ }
+ plan, directory := prepareExpired()
+ if err := CancelDataImport(plan.ID); err != nil {
+ t.Fatalf("cannot cancel an expired preparation: %v", err)
+ }
+ if _, err := os.Stat(directory); !os.IsNotExist(err) {
+ t.Fatal("expired cancellation left the uploaded archive")
+ }
+ _, directory = prepareExpired()
+ if _, err := PrepareDataImport(dataImportTestZip(t, source)); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(directory); !os.IsNotExist(err) {
+ t.Fatal("new preparation did not reclaim an expired preparation")
+ }
+ if err := recoverDataImports(); err != nil {
+ t.Fatal(err)
+ }
+ // 已提交的恢复材料不受过期清理影响。
+ committed, err := PrepareDataImport(dataImportTestZip(t, source))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err = CommitDataImport(committed.ID, "skip", ""); err != nil {
+ t.Fatal(err)
+ }
+ committedDir, _ := dataImportDirectory(committed.ID)
+ stageData, err := os.ReadFile(filepath.Join(committedDir, "stage.json"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var committedStage dataImportStage
+ if err = json.Unmarshal(stageData, &committedStage); err != nil {
+ t.Fatal(err)
+ }
+ committedStage.Created = time.Now().Add(-time.Hour).Unix()
+ if err = writeDataImportJSON(filepath.Join(committedDir, "stage.json"), committedStage); err != nil {
+ t.Fatal(err)
+ }
+ if err = recoverDataImports(); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = os.Stat(filepath.Join(committedDir, "source.zip")); err != nil {
+ t.Fatalf("cleanup discarded committed recovery material: %v", err)
+ }
+}
+
+func TestDataImportFailedRollbackQuarantinesWritesUntilRecovery(t *testing.T) {
+ notebookArchiveTestWorkspace(t)
+ previousPending, previousReadonly := dataImportRecoveryRequired.Load(), util.ReadOnly
+ util.ReadOnly = false
+ t.Cleanup(func() { dataImportRecoveryRequired.Store(previousPending); util.ReadOnly = previousReadonly })
+ directory, _ := dataImportDirectory("20261020230000-reviewa")
+ install := filepath.Join(directory, "install")
+ for _, name := range []string{"a.txt", "b.txt"} {
+ if err := writeDataImportBytes(filepath.Join(install, name), []byte("new")); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(util.DataDir, name), []byte("original"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ backup := filepath.Join(directory, "rollback", "a.txt")
+ writes := 0
+ err := applyDataImport(directory, install, func(source, target string) error {
+ writes++
+ if writes == 2 {
+ if err := os.Rename(backup, backup+".unavailable"); err != nil {
+ t.Fatal(err)
+ }
+ return errors.New("write failure with temporarily unavailable rollback source")
+ }
+ return copyDataImportFile(source, target)
+ })
+ if err == nil || !dataImportRecoveryRequired.Load() {
+ t.Fatal("failed rollback did not keep write quarantine")
+ }
+ engine := gin.New()
+ engine.Use(func(c *gin.Context) { c.Set(RoleContextKey, RoleAdministrator) })
+ entered := false
+ engine.POST("/api/test/write", CheckReadonly, func(c *gin.Context) { entered = true; c.Status(http.StatusNoContent) })
+ writeRequest := func() {
+ engine.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodPost, "/api/test/write", nil))
+ }
+ writeRequest()
+ if entered {
+ t.Fatal("admitted another write after failed rollback")
+ }
+ if err = AcquireEncryptedBoxOperation(""); err == nil {
+ t.Fatal("admitted a new notebook operation during pending recovery")
+ }
+ if _, err = deriveKEK("any-password"); err == nil {
+ t.Fatal("admitted key adoption during pending recovery")
+ }
+ if _, _, err = CreateRepoSnapshot("must not capture a partial import"); err == nil {
+ t.Fatal("allowed a snapshot of a partially installed batch")
+ }
+ for _, syncAttempt := range []func() error{
+ func() error { return syncDataLocked(false, false) }, syncRepoDownload, syncRepoUpload, bootSyncRepo,
+ func() error { _, err := syncRepo(false, false); return err },
+ } {
+ if err = syncAttempt(); err == nil {
+ t.Fatal("allowed sync of a partially installed batch")
+ }
+ }
+ source := t.TempDir()
+ if err = os.WriteFile(filepath.Join(source, "next.txt"), []byte("next"), 0600); err != nil {
+ t.Fatal(err)
+ }
+ archive := dataImportTestZip(t, source)
+ if _, err = PrepareDataImport(archive); err == nil || !dataImportRecoveryRequired.Load() {
+ t.Fatal("failed recovery reopened admission")
+ }
+ if err = os.Rename(backup+".unavailable", backup); err != nil {
+ t.Fatal(err)
+ }
+ if _, err = PrepareDataImport(archive); err != nil {
+ t.Fatalf("preparation could not retry recovery safely: %v", err)
+ }
+ if dataImportRecoveryRequired.Load() {
+ t.Fatal("successful recovery did not reopen admission")
+ }
+ content, err := os.ReadFile(filepath.Join(util.DataDir, "a.txt"))
+ if err != nil || string(content) != "original" {
+ t.Fatalf("original data was not recovered: %v", err)
+ }
+ writeRequest()
+ if !entered {
+ t.Fatal("write admission remained blocked after recovery")
+ }
}
func TestDataImportTransactionFailureAndRestartRecovery(t *testing.T) {
@@ -577,7 +836,7 @@ func TestDataImportTransactionFailureAndRestartRecovery(t *testing.T) {
notebookArchiveTestWorkspace(t)
directory, _ := dataImportDirectory("20261020180002-importc")
install := filepath.Join(directory, "install")
- for name, content := range map[string]string{"a.txt": "new", "new/b.txt": "new"} {
+ for name, content := range map[string]string{"a.txt": "new", "new/b.txt": "new", "new/c.txt": "new"} {
if err := writeDataImportBytes(filepath.Join(install, filepath.FromSlash(name)), []byte(content)); err != nil {
t.Fatal(err)
}
@@ -599,7 +858,7 @@ func TestDataImportTransactionFailureAndRestartRecovery(t *testing.T) {
}()
err = applyDataImport(directory, install, func(source, target string) error {
writes++
- if writes == 2 {
+ if writes == 3 {
if crash {
panic(crashed)
}
diff --git a/kernel/model/import_data_transaction.go b/kernel/model/import_data_transaction.go
index 17733ef66a..8e18bd2a8f 100644
--- a/kernel/model/import_data_transaction.go
+++ b/kernel/model/import_data_transaction.go
@@ -26,8 +26,10 @@ import (
"slices"
"sort"
"strings"
+ "time"
"github.com/88250/lute/ast"
+ "github.com/siyuan-note/logging"
"github.com/siyuan-note/siyuan/kernel/util"
)
@@ -151,6 +153,7 @@ func applyDataImport(directory, install string, write func(string, string) error
if err = writeDataImportJSON(filepath.Join(directory, "operation.json"), operation); err != nil {
return err
}
+ dataImportRecoveryRequired.Store(true)
for _, entry := range operation.Files {
target, targetErr := dataImportTarget(entry.Path)
if targetErr == nil {
@@ -163,17 +166,24 @@ func applyDataImport(directory, install string, write func(string, string) error
targetErr = write(filepath.Join(install, filepath.FromSlash(entry.Path)), target)
}
if targetErr != nil {
- return errors.Join(targetErr, rollbackDataImport(directory, operation))
+ return abortDataImport(directory, operation, targetErr)
}
}
operation.State = "committed"
if err = writeDataImportJSON(filepath.Join(directory, "operation.json"), operation); err != nil {
operation.State = "committing"
- return errors.Join(err, rollbackDataImport(directory, operation))
+ return abortDataImport(directory, operation, err)
}
+ dataImportRecoveryRequired.Store(false)
return nil
}
+func abortDataImport(directory string, operation *dataImportOperation, cause error) error {
+ err := rollbackDataImport(directory, operation)
+ dataImportRecoveryRequired.Store(err != nil)
+ return errors.Join(cause, err)
+}
+
func rollbackDataImport(directory string, operation *dataImportOperation) error {
for _, entry := range slices.Backward(operation.Files) {
target, err := dataImportTarget(entry.Path)
@@ -217,11 +227,15 @@ func rollbackDataImport(directory string, operation *dataImportOperation) error
}
}
operation.State = "rolledback"
- return writeDataImportJSON(filepath.Join(directory, "operation.json"), operation)
+ if err := writeDataImportJSON(filepath.Join(directory, "operation.json"), operation); err != nil {
+ return err
+ }
+ return nil
}
// recoverDataImports 在挂载和同步前补偿未完成的批次,成功批次的原件和恢复材料继续保留。
-func recoverDataImports() error {
+func recoverDataImports() (err error) {
+ defer func() { dataImportRecoveryRequired.Store(err != nil) }()
entries, err := os.ReadDir(dataImportBase())
if os.IsNotExist(err) {
return nil
@@ -255,10 +269,50 @@ func recoverDataImports() error {
seen[strings.ToLower(file.Path)] = true
}
if operation.State == "committing" {
+ dataImportRecoveryRequired.Store(true)
if err = rollbackDataImport(directory, operation); err != nil {
return fmt.Errorf("%s: %w", Conf.Language(424), err)
}
}
}
+ if cleanupErr := cleanupExpiredDataImports(); cleanupErr != nil {
+ logging.LogWarnf("clean up expired Data.zip preparations failed: %s", cleanupErr)
+ }
+ return nil
+}
+
+// recoverDataImportsForPrepare 在独占写准入、同步和密钥锁下恢复,预检自身不持共享锁执行补偿写入。
+func recoverDataImportsForPrepare() error {
+ dataImportAccess.Lock()
+ defer dataImportAccess.Unlock()
+ dataImportMu.Lock()
+ defer dataImportMu.Unlock()
+ lockSync()
+ defer unlockSync()
+ notebookCryptoMu.Lock()
+ defer notebookCryptoMu.Unlock()
+ return recoverDataImports()
+}
+
+func cleanupExpiredDataImports() error {
+ entries, err := os.ReadDir(dataImportBase())
+ if os.IsNotExist(err) {
+ return nil
+ }
+ if err != nil {
+ return err
+ }
+ for _, entry := range entries {
+ if !entry.IsDir() || !ast.IsNodeIDPattern(entry.Name()) {
+ continue
+ }
+ directory, stage, readErr := readDataImportStage(entry.Name())
+ if readErr != nil || time.Now().Unix()-stage.Created <= int64(dataImportTTL.Seconds()) {
+ continue
+ }
+ if err = os.RemoveAll(directory); err != nil {
+ return err
+ }
+ }
return nil
}
diff --git a/kernel/model/repository.go b/kernel/model/repository.go
index 36edc39625..73d8f9123c 100644
--- a/kernel/model/repository.go
+++ b/kernel/model/repository.go
@@ -1875,6 +1875,9 @@ func normalizeSnapshotMemo(memo string) string {
}
func CreateRepoSnapshot(memo string) (id string, created bool, err error) {
+ if err = checkDataImportRecovery(); err != nil {
+ return
+ }
if 1 > len(Conf.Repo.Load().Key) {
err = errors.New(Conf.Language(26))
return
@@ -2010,6 +2013,9 @@ func formatSyncRepoErrorMsg(err error) string {
}
func syncRepoDownload() (err error) {
+ if err = checkDataImportRecovery(); err != nil {
+ return
+ }
if 1 > len(Conf.Repo.Load().Key) {
planSyncAfter(fixSyncInterval)
@@ -2085,6 +2091,9 @@ func syncRepoDownload() (err error) {
}
func syncRepoUpload() (err error) {
+ if err = checkDataImportRecovery(); err != nil {
+ return
+ }
if 1 > len(Conf.Repo.Load().Key) {
planSyncAfter(fixSyncInterval)
@@ -2160,6 +2169,9 @@ func syncRepoUpload() (err error) {
var isBootSyncing = atomic.Bool{}
func bootSyncRepo() (err error) {
+ if err = checkDataImportRecovery(); err != nil {
+ return
+ }
if 1 > len(Conf.Repo.Load().Key) {
autoSyncErrCount.Add(1)
planSyncAfter(fixSyncInterval)
@@ -2302,6 +2314,9 @@ func bootSyncRepo() (err error) {
}
func syncRepo(exit, byHand bool) (cloudPublished bool, err error) {
+ if err = checkDataImportRecovery(); err != nil {
+ return
+ }
if 1 > len(Conf.Repo.Load().Key) {
autoSyncErrCount.Add(1)
planSyncAfter(fixSyncInterval)
diff --git a/kernel/model/session.go b/kernel/model/session.go
index 284df2b006..a7265f80fc 100644
--- a/kernel/model/session.go
+++ b/kernel/model/session.go
@@ -195,9 +195,16 @@ func CheckReadonly(c *gin.Context) {
return
}
// 导入提交期间暂停其他写接口,避免已确认的磁盘快照被并发请求修改。
- if c.Request.URL.Path != "/api/import/commitDataImport" {
+ if c.Request.URL.Path != "/api/import/commitDataImport" && c.Request.URL.Path != "/api/import/prepareDataImport" {
dataImportAccess.RLock()
defer dataImportAccess.RUnlock()
+ if err := checkDataImportRecovery(); err != nil {
+ result := util.NewResult()
+ result.Code, result.Msg = -1, err.Error()
+ c.JSON(http.StatusOK, result)
+ c.Abort()
+ return
+ }
c.Next()
}
}
diff --git a/kernel/model/sync.go b/kernel/model/sync.go
index 45b6fc6a38..278deee086 100644
--- a/kernel/model/sync.go
+++ b/kernel/model/sync.go
@@ -326,6 +326,9 @@ func syncData(exit, byHand bool) (err error) {
}
func syncDataLocked(exit, byHand bool) error {
+ if err := checkDataImportRecovery(); err != nil {
+ return err
+ }
revision := pendingSync.begin()
util.BroadcastByType("main", "syncing", 0, Conf.Language(81), nil)
if exit {