Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

18.8 KB
Raw Blame
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later

package model

import (
	"archive/zip"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"reflect"
	"slices"
	"sort"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/88250/lute/ast"
	"github.com/siyuan-note/siyuan/kernel/conf"
	"github.com/siyuan-note/siyuan/kernel/util"
)

const dataImportTTL = 30 * time.Minute

var dataImportMu sync.Mutex
var dataImportAccess sync.RWMutex
var dataImportRecoveryRequired atomic.Bool

func checkDataImportRecovery() error {
	if dataImportRecoveryRequired.Load() {
		return errors.New(Conf.Language(424))
	}
	return nil
}

type DataImportPlan struct {
	ID        string   `json:"id"`
	Conflicts []string `json:"conflicts"`
	Encrypted []string `json:"encrypted"`
}

type dataImportStage struct {
	Version int                            `json:"version"`
	Created int64                          `json:"created"`
	Plan    DataImportPlan                 `json:"plan"`
	Files   map[string]notebookArchiveFile `json:"files"`
}

func dataImportBase() string { return filepath.Join(util.ConfDir, "data-imports") }

func dataImportDirectory(id string) (string, error) {
	if !ast.IsNodeIDPattern(id) {
		return "", errors.New(Conf.Language(422))
	}
	directory := filepath.Join(dataImportBase(), id)
	if err := validateEncryptedCleanupPath(util.ConfDir, directory); err != nil {
		return "", err
	}
	return directory, nil
}

// PrepareDataImport 先在独占访问下恢复未完成批次,再保存上传原件和预检结果,不安装上传内容。
func PrepareDataImport(zipPath string) (plan DataImportPlan, err error) {
	if err = recoverDataImportsForPrepare(); err != nil {
		return
	}
	dataImportMu.Lock()
	defer dataImportMu.Unlock()
	if err = checkDataImportRecovery(); err != nil {
		return
	}
	plan = DataImportPlan{ID: ast.NewNodeID(), Conflicts: []string{}, Encrypted: []string{}}
	directory, err := dataImportDirectory(plan.ID)
	if err != nil {
		return
	}
	defer func() {
		if err != nil {
			err = errors.Join(err, os.RemoveAll(directory))
		}
	}()
	if err = archiveCopyFile(zipPath, filepath.Join(directory, "source.zip")); err != nil {
		return
	}
	incoming := filepath.Join(directory, "incoming")
	if err = extractDataImport(filepath.Join(directory, "source.zip"), incoming); err != nil {
		return
	}
	identities, err := inspectDataImportNotebooks(incoming)
	if err != nil {
		return
	}
	plan = planDataImport(plan.ID, identities)
	files, err := archiveDirectoryFiles(incoming)
	if err != nil {
		return
	}
	stage := dataImportStage{Version: 1, Created: time.Now().Unix(), Plan: plan, Files: files}
	err = writeDataImportJSON(filepath.Join(directory, "stage.json"), stage)
	return
}

func planDataImport(id string, identities map[string]*conf.BoxEncryption) DataImportPlan {
	plan := DataImportPlan{ID: id, Conflicts: []string{}, Encrypted: []string{}}
	for boxID, crypt := range identities {
		if crypt != nil {
			plan.Encrypted = append(plan.Encrypted, boxID)
		}
		if _, err := os.Lstat(filepath.Join(util.DataDir, boxID)); err == nil && (crypt != nil || IsEncryptedBox(boxID)) {
			plan.Conflicts = append(plan.Conflicts, boxID)
		}
	}
	sort.Strings(plan.Conflicts)
	sort.Strings(plan.Encrypted)
	return plan
}

// extractDataImport 保留原有单层 Data.zip 容器,拒绝重复路径、跨平台歧义和链接条目。
func extractDataImport(zipPath, destination string) error {
	reader, err := zip.OpenReader(zipPath)
	if err != nil {
		return err
	}
	defer reader.Close()
	if len(reader.File) == 0 || len(reader.File) > notebookArchiveMaxEntries {
		return errors.New(Conf.Language(198))
	}
	root := ""
	seen := map[string]bool{}
	for _, file := range reader.File {
		name := strings.TrimSuffix(file.Name, "/")
		if !archiveRelativePath(name) || !file.Mode().IsRegular() && !file.FileInfo().IsDir() {
			return errors.New(Conf.Language(198))
		}
		parts := strings.SplitN(name, "/", 2)
		if root == "" {
			root = parts[0]
		}
		if parts[0] != root || len(parts) == 1 && !file.FileInfo().IsDir() {
			return errors.New(Conf.Language(198))
		}
		if seen[strings.ToLower(name)] {
			return errors.New(Conf.Language(198))
		}
		seen[strings.ToLower(name)] = true
		if len(parts) == 1 || file.FileInfo().IsDir() {
			continue
		}
		target := filepath.Join(destination, filepath.FromSlash(parts[1]))
		if err = os.MkdirAll(filepath.Dir(target), 0700); err != nil {
			return err
		}
		input, openErr := file.Open()
		if openErr != nil {
			return openErr
		}
		output, createErr := os.OpenFile(target, os.O_CREATE|os.O_EXCL|os.O_WRONLY, 0600)
		if createErr != nil {
			input.Close()
			return createErr
		}
		_, copyErr := io.Copy(output, input)
		inputErr, outputErr := input.Close(), output.Close()
		if err = errors.Join(copyErr, inputErr, outputErr); err != nil {
			return err
		}
	}
	return nil
}

// inspectDataImportNotebooks 同时记录普通与加密身份,避免普通来源覆盖现有加密目录。
func inspectDataImportNotebooks(root string) (map[string]*conf.BoxEncryption, error) {
	entries, err := os.ReadDir(root)
	if err != nil {
		return nil, err
	}
	identities := map[string]*conf.BoxEncryption{}
	if err = validateDataImportNotebookPaths(root, entries); err != nil {
		return nil, err
	}
	for _, entry := range entries {
		if strings.HasSuffix(entry.Name(), ".sy") {
			return nil, errors.New(Conf.Language(198))
		}
		if !entry.IsDir() || !ast.IsNodeIDPattern(entry.Name()) {
			continue
		}
		boxID := entry.Name()
		directory := filepath.Join(root, boxID)
		data, readErr := os.ReadFile(filepath.Join(directory, ".siyuan", "conf.json"))
		var boxConf *conf.BoxConf
		if readErr == nil {
			boxConf = conf.NewBoxConf()
			if err = json.Unmarshal(data, boxConf); err != nil {
				return nil, err
			}
		} else if !os.IsNotExist(readErr) {
			return nil, readErr
		}
		var backup *conf.BoxEncryption
		backupPath := filepath.Join(directory, ".siyuan", notebookCryptoBackupFilename)
		if _, statErr := os.Lstat(backupPath); statErr == nil {
			backup, err = readBoxEncryptionFile(backupPath)
			if err != nil {
				return nil, err
			}
		} else if !os.IsNotExist(statErr) {
			return nil, statErr
		}
		var crypt *conf.BoxEncryption
		if boxConf != nil && boxConf.Encrypted {
			crypt = boxConf.BoxCrypt
			if crypt == nil || validateBoxEncryption(crypt) != nil {
				crypt = backup
			}
			if crypt == nil {
				return nil, errors.New(Conf.Language(316))
			}
		} else if boxConf != nil && backup != nil {
			return nil, fmt.Errorf(Conf.Language(423), boxID)
		} else {
			crypt = backup
		}
		payload, payloadErr := hasEncryptedNotebookPayloadAtPath(directory)
		if payloadErr != nil {
			return nil, payloadErr
		}
		if crypt == nil && payload {
			return nil, errors.New(Conf.Language(316))
		}
		if crypt != nil {
			if err = validateBoxEncryption(crypt); err != nil {
				return nil, err
			}
		}
		identities[boxID] = crypt
	}
	return identities, nil
}

// validateDataImportNotebookPaths 拒绝非规范 ID,并逐级核对已有目录项,防止大小写和短文件名别名绕过身份检查。
func validateDataImportNotebookPaths(root string, entries []os.DirEntry) error {
	for _, entry := range entries {
		if name := entry.Name(); ast.IsNodeIDPattern(strings.ToLower(name)) && !ast.IsNodeIDPattern(name) {
			return errors.New(Conf.Language(198))
		}
	}
	directories := map[string]map[string]bool{}
	return archiveWalk(root, func(name, _ string) error {
		parent := util.DataDir
		for _, part := range strings.Split(name, "/") {
			names, cached := directories[parent]
			if !cached {
				children, err := os.ReadDir(parent)
				if os.IsNotExist(err) {
					return nil
				}
				if err != nil {
					return err
				}
				names = map[string]bool{}
				for _, child := range children {
					names[child.Name()] = true
				}
				directories[parent] = names
			}
			parent = filepath.Join(parent, part)
			if names[part] {
				continue
			}
			if _, err := os.Lstat(parent); err == nil {
				return errors.New(Conf.Language(198))
			} else if !os.IsNotExist(err) {
				return err
			}
			return nil
		}
		return nil
	})
}

func loadDataImportStage(id string) (string, *dataImportStage, error) {
	directory, stage, err := readDataImportStage(id)
	if err != nil {
		return "", nil, err
	}
	if time.Now().Unix()-stage.Created > int64(dataImportTTL.Seconds()) || stage.Created > time.Now().Unix() {
		return "", nil, errors.New(Conf.Language(422))
	}
	return directory, stage, nil
}

func readDataImportStage(id string) (string, *dataImportStage, error) {
	directory, err := dataImportDirectory(id)
	if err != nil {
		return "", nil, err
	}
	data, err := os.ReadFile(filepath.Join(directory, "stage.json"))
	stage := &dataImportStage{}
	if err != nil || json.Unmarshal(data, stage) != nil || stage.Version != 1 || stage.Plan.ID != id || stage.Created <= 0 {
		return "", nil, errors.New(Conf.Language(422))
	}
	if _, err = os.Lstat(filepath.Join(directory, "operation.json")); !os.IsNotExist(err) {
		return "", nil, errors.New(Conf.Language(422))
	}
	return directory, stage, nil
}

func CancelDataImport(id string) error {
	dataImportMu.Lock()
	defer dataImportMu.Unlock()
	directory, _, err := readDataImportStage(id)
	if err != nil {
		return err
	}
	return os.RemoveAll(directory)
}

// CommitDataImport 在预检全部通过后提交按路径覆盖;跳过的目录与密钥材料保持原样。
func CommitDataImport(id, mode, password string) (skipped []string, err error) {
	dataImportAccess.Lock()
	defer dataImportAccess.Unlock()
	dataImportMu.Lock()
	defer dataImportMu.Unlock()
	lockSync()
	defer unlockSync()
	notebookCryptoMu.Lock()
	defer notebookCryptoMu.Unlock()
	if mode != "skip" && mode != "overwrite" {
		return nil, errors.New(Conf.Language(422))
	}
	if err = recoverDataImports(); err != nil {
		return
	}
	if err = checkPendingNotebookArchives(); err != nil {
		return
	}
	if migration, migrationErr := readMasterPasswordMigration(); migrationErr != nil || migration != nil {
		return nil, errors.New(Conf.Language(239))
	}
	directory, stage, err := loadDataImportStage(id)
	if err != nil {
		return
	}
	incoming := filepath.Join(directory, "incoming")
	files, err := archiveDirectoryFiles(incoming)
	if err != nil || !reflect.DeepEqual(files, stage.Files) {
		return nil, errors.New(Conf.Language(422))
	}
	identities, err := inspectDataImportNotebooks(incoming)
	if err != nil {
		return
	}
	if !reflect.DeepEqual(planDataImport(id, identities), stage.Plan) {
		return nil, errors.New(Conf.Language(422))
	}
	skipped = []string{}
	if mode == "skip" {
		skipped = append(skipped, stage.Plan.Conflicts...)
	}
	var encrypted []string
	for _, boxID := range stage.Plan.Encrypted {
		if !slices.Contains(skipped, boxID) {
			encrypted = append(encrypted, boxID)
		}
	}
	for _, boxID := range stage.Plan.Conflicts {
		if !slices.Contains(skipped, boxID) && (identities[boxID] == nil || !IsEncryptedBox(boxID)) {
			return nil, fmt.Errorf(Conf.Language(423), boxID)
		}
	}
	var nc *conf.NotebookCrypto
	var kek []byte
	if len(encrypted) != 0 {
		nc, kek, err = dataImportKey(incoming, password, encrypted, identities)
		if err != nil {
			return
		}
		defer zeroAndClear(kek)
	}
	// 生命周期锁先阻止新的访问并排空既有请求,随后才冻结编辑队列和复制磁盘快照。
	for _, boxID := range encrypted {
		release := holdEncryptedBoxTransition(boxID)
		defer release()
		if IsEncryptedBox(boxID) {
			mounted := isEncryptedBoxMounted(boxID)
			lockBoxWithPreparationHeld(boxID, func() {
				if mounted {
					GenerateFileHistoryForBox(&Box{ID: boxID})
				}
			})
		}
	}
	FlushTxQueue()
	flushLock.Lock()
	defer flushLock.Unlock()
	if txQueueSize() != 0 {
		return nil, errors.New(Conf.Language(239))
	}
	install := filepath.Join(directory, "install")
	if err = validateEncryptedCleanupPath(directory, install); err != nil {
		return
	}
	if err = os.RemoveAll(install); err != nil {
		return
	}
	if err = os.MkdirAll(install, 0700); err != nil {
		return
	}
	if err = stageDataImportFiles(incoming, install, skipped); err != nil {
		return
	}
	for _, boxID := range encrypted {
		if err = stageEncryptedDataImport(incoming, install, boxID, identities[boxID], nc, kek); err != nil {
			return
		}
	}
	if nc != nil {
		nc.Enabled = true
		prepareBackupForWrite(nc)
		nc.KEKMAC = computeKEKMAC(nc, kek)
		if err = writeDataImportJSON(filepath.Join(install, ".siyuan", "data-crypto-backup.json"), nc); err != nil {
			return
		}
		if !verifyKEKAgainstExistingBoxes(kek, nc) || !verifyKEKAgainstEncryptedHistory(kek, nc) {
			return nil, errors.New(Conf.Language(316))
		}
	}
	keys, err := notebookArchiveKeys()
	if err != nil {
		return nil, err
	}
	for name, data := range keys {
		if err = writeDataImportBytes(filepath.Join(directory, name), data); err != nil {
			return nil, err
		}
	}
	if err = applyDataImport(directory, install, copyDataImportFile); err != nil {
		return
	}
	if nc != nil {
		Conf.m.Lock()
		Conf.NotebookCrypto = nc
		Conf.m.Unlock()
		Conf.Save()
	}
	for _, boxID := range encrypted {
		forgetRuntimeNormalBox(boxID)
		markRuntimeEncryptedBox(boxID)
		setEncryptedBoxState(boxID, EncryptedBoxStateLocked)
	}
	IncSync()
	return
}

func stageDataImportFiles(source, target string, skipped []string) error {
	return archiveWalk(source, func(name, fullPath string) error {
		lower := strings.ToLower(name)
		if slices.Contains(skipped, strings.SplitN(name, "/", 2)[0]) || lower == ".siyuan/data-crypto-backup.json" ||
			lower == ".siyuan/master-password-migration.json" {
			return nil
		}
		return copyDataImportFile(fullPath, filepath.Join(target, filepath.FromSlash(name)))
	})
}

func stageEncryptedDataImport(incoming, install, boxID string, crypt *conf.BoxEncryption, nc *conf.NotebookCrypto, kek []byte) error {
	dek, err := decryptWrappedDEKWithHistory(boxID, crypt, kek, nc)
	if err != nil {
		return errors.New(Conf.Language(316))
	}
	defer zeroAndClear(dek)
	if err = authenticateNotebookArchiveBox(filepath.Join(incoming, boxID), boxID, dek, crypt); err != nil {
		return err
	}
	current := filepath.Join(util.DataDir, boxID)
	merged := filepath.Join(install, boxID)
	if _, statErr := os.Lstat(current); statErr == nil {
		local, readErr := GetBoxEncryption(boxID)
		if readErr != nil || local == nil {
			return fmt.Errorf(Conf.Language(423), boxID)
		}
		localDEK, decryptErr := decryptWrappedDEKWithHistory(boxID, local, kek, nc)
		same := decryptErr == nil && bytes.Equal(localDEK, dek)
		zeroAndClear(localDEK)
		if !same {
			return fmt.Errorf(Conf.Language(423), boxID)
		}
		// 先保留本机独有路径,再覆盖来源路径;不把旧文件隐式删除为镜像恢复。
		if err = archiveWalk(current, func(name, p string) error {
			target := filepath.Join(merged, filepath.FromSlash(name))
			if _, existsErr := os.Lstat(target); existsErr == nil {
				return nil
			} else if !os.IsNotExist(existsErr) {
				return existsErr
			}
			return copyDataImportFile(p, target)
		}); err != nil {
			return err
		}
	} else if !os.IsNotExist(statErr) {
		return statErr
	}
	if err = authenticateNotebookArchiveBox(merged, boxID, dek, crypt); err != nil {
		return err
	}
	boxConf := conf.NewBoxConf()
	if data, readErr := os.ReadFile(filepath.Join(merged, ".siyuan", "conf.json")); readErr == nil {
		if err = json.Unmarshal(data, boxConf); err != nil {
			return err
		}
	} else if !os.IsNotExist(readErr) {
		return readErr
	}
	boxConf.Encrypted, boxConf.Closed, boxConf.BoxCrypt = true, true, crypt
	clearBoxMetadata(boxConf)
	if err = writeDataImportJSON(filepath.Join(merged, ".siyuan", "conf.json"), boxConf); err != nil {
		return err
	}
	return writeDataImportJSON(filepath.Join(merged, ".siyuan", notebookCryptoBackupFilename), crypt)
}

// dataImportKey 优先使用本机认证配置;候选必须保留当前及历史 KEK,避免旧备份回退密码。
func dataImportKey(incoming, password string, encrypted []string, identities map[string]*conf.BoxEncryption) (*conf.NotebookCrypto, []byte, error) {
	current := currentNotebookCrypto()
	candidates := []*conf.NotebookCrypto{current}
	if data, err := os.ReadFile(filepath.Join(incoming, ".siyuan", "data-crypto-backup.json")); err == nil {
		candidate := &conf.NotebookCrypto{}
		if json.Unmarshal(data, candidate) == nil {
			candidates = append(candidates, candidate)
		}
	}
	for _, nc := range candidates {
		if !notebookCryptoConfigurationComplete(nc) || password == "" {
			continue
		}
		kek := util.DeriveKey(password, nc.MasterSalt, nc.KDFParams)
		verifier, err := util.DecryptWithAAD(kek, nc.KEKVerifier, []byte("siyuan:kek-verifier"))
		if err != nil || !bytes.Equal(verifier, kekVerifierMagic) || !verifyKEKMAC(nc, kek) ||
			!dataImportPreservesKeys(current, nc, kek) || !verifyKEKAgainstExistingBoxes(kek, nc) || !verifyKEKAgainstEncryptedHistory(kek, nc) {
			zeroAndClear(kek)
			continue
		}
		valid := true
		for _, id := range encrypted {
			dek, decryptErr := decryptWrappedDEKWithHistory(id, identities[id], kek, nc)
			if decryptErr == nil {
				decryptErr = authenticateNotebookArchiveBox(filepath.Join(incoming, id), id, dek, identities[id])
			}
			if decryptErr == nil && IsEncryptedBox(id) {
				local, localErr := GetBoxEncryption(id)
				if localErr != nil || local == nil {
					decryptErr = errors.New(Conf.Language(316))
				} else {
					localDEK, localDecryptErr := decryptWrappedDEKWithHistory(id, local, kek, nc)
					if localDecryptErr != nil || !bytes.Equal(localDEK, dek) {
						decryptErr = errors.New(Conf.Language(316))
					}
					zeroAndClear(localDEK)
				}
			}
			zeroAndClear(dek)
			if decryptErr != nil {
				valid = false
				break
			}
		}
		if !valid {
			zeroAndClear(kek)
			continue
		}
		nc.AutoLockMinutes = current.AutoLockMinutes
		return nc, kek, nil
	}
	return nil, nil, errors.New(Conf.Language(316))
}

func dataImportPreservesKeys(current, candidate *conf.NotebookCrypto, kek []byte) bool {
	keys, err := decryptHistoryKEKs(kek, candidate.HistoryKEKs)
	if err != nil {
		return false
	}
	defer func() { clearHistoryKEKs(keys) }()
	if !notebookCryptoConfigurationComplete(current) {
		return NotebookCryptoLifecycleState(false) == NotebookCryptoStateDisabled
	}
	if !bytes.Equal(current.MasterSalt, candidate.MasterSalt) {
		return false
	}
	keys = append(keys, bytes.Clone(kek))
	for _, key := range keys {
		verifier, decryptErr := util.DecryptWithAAD(key, current.KEKVerifier, []byte("siyuan:kek-verifier"))
		if decryptErr != nil || !bytes.Equal(verifier, kekVerifierMagic) || !verifyKEKMAC(current, key) {
			continue
		}
		previous, historyErr := decryptHistoryKEKs(key, current.HistoryKEKs)
		defer clearHistoryKEKs(previous)
		if historyErr != nil {
			return false
		}
		for _, old := range previous {
			if !slices.ContainsFunc(keys, func(kept []byte) bool { return bytes.Equal(kept, old) }) {
				return false
			}
		}
		return true
	}
	return false
}

func writeDataImportJSON(p string, value interface{}) error {
	data, err := json.Marshal(value)
	if err != nil {
		return err
	}
	return writeDataImportBytes(p, data)
}

func writeDataImportBytes(p string, data []byte) error {
	if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil {
		return err
	}
	return writeEncryptedCleanupData(p, data)
}