Skip to content

An open-source, privacy-first, self-hosted knowledge workspace where humans and AI agents work together 开源、隐私优先、自托管的知识工作空间,让人与智能体在此协作

11.1 KB
Raw Blame
// SiYuan - https://github.com/siyuan-note/siyuan
// Copyright (c) 2020-present, b3log.org
// SPDX-License-Identifier: AGPL-3.0-or-later

package bazaar

import (
	"archive/zip"
	"bytes"
	"context"
	"crypto/sha256"
	"errors"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strings"

	"github.com/88250/gulu"
)

const (
	// MaxLocalPackageArchiveSize 限制上传的本地集市包压缩文件大小。
	MaxLocalPackageArchiveSize int64  = 128 * 1024 * 1024
	maxLocalPackageFileCount          = 10000
	maxLocalPackageFileSize    uint64 = 256 * 1024 * 1024
	maxLocalPackageExtractSize uint64 = 512 * 1024 * 1024
)

var localPackageManifests = map[string]string{
	"plugin.json":   "plugins",
	"theme.json":    "themes",
	"icon.json":     "icons",
	"template.json": "templates",
	"widget.json":   "widgets",
}

// IsValidPackageType 判断 pkgType 是否为集市支持的包类型(复数形式),以本地包清单映射为准。
func IsValidPackageType(pkgType string) bool {
	for _, t := range localPackageManifests {
		if t == pkgType {
			return true
		}
	}
	return false
}

// ExtractLocalPackage 将本地集市包解压到临时目录并识别包类型。
func ExtractLocalPackage(archivePath string) (pkgType string, pkg *Package, packagePath string, cleanup func(), err error) {
	pkgType, pkg, packagePath, _, cleanup, err = extractLocalPackageWithHash(context.Background(), archivePath, "")
	return
}

// ExtractLocalPackageWithHash 从同一份已核验字节解压,不会在摘要验证后重新打开原始路径。
func ExtractLocalPackageWithHash(archivePath, expectedHash string) (pkgType string, pkg *Package, packagePath, packageHash string, cleanup func(), err error) {
	return ExtractLocalPackageWithHashContext(context.Background(), archivePath, expectedHash)
}

// ExtractLocalPackageWithHashContext 在核验与解压期间响应取消,未提交的临时内容会清理。
// 解压始终落在 temp,校验通过后才由安装流程拷入 data。
func ExtractLocalPackageWithHashContext(ctx context.Context, archivePath, expectedHash string) (pkgType string, pkg *Package, packagePath, packageHash string, cleanup func(), err error) {
	return extractLocalPackageWithHash(ctx, archivePath, expectedHash)
}

func extractLocalPackageWithHash(ctx context.Context, archivePath, expectedHash string) (pkgType string, pkg *Package, packagePath, packageHash string, cleanup func(), err error) {
	cleanup = func() {}
	if err = ctx.Err(); err != nil {
		return
	}
	if expectedHash != "" && !validInstallHash(expectedHash) {
		err = errors.New("invalid package hash")
		return
	}
	data, err := readInstallArchive(archivePath)
	if err != nil {
		return
	}
	packageHash = fmt.Sprintf("%x", sha256.Sum256(data))
	if expectedHash != "" && packageHash != expectedHash {
		err = ErrPackageHashConflict
		return
	}
	if err = ctx.Err(); err != nil {
		return
	}
	tempPath, err := makePackageExtractionDir("local", "extract-")
	if err != nil {
		return
	}
	cleanup = func() { _ = os.RemoveAll(tempPath) }
	reader, readerErr := zip.NewReader(bytes.NewReader(data), int64(len(data)))
	if readerErr != nil {
		err = errors.New("invalid marketplace package archive")
		cleanup()
		return
	}
	if err = extractLocalPackageReaderContext(ctx, reader, tempPath); err != nil {
		cleanup()
		return
	}

	packagePath, err = localPackageRoot(tempPath)
	if err != nil {
		cleanup()
		return
	}

	var manifestPath string
	for manifestName, packageType := range localPackageManifests {
		candidate := filepath.Join(packagePath, manifestName)
		if info, statErr := os.Stat(candidate); statErr == nil && info.Mode().IsRegular() {
			if manifestPath != "" {
				err = errors.New("multiple marketplace package manifests found")
				cleanup()
				return
			}
			pkgType = packageType
			manifestPath = candidate
		}
	}
	if manifestPath == "" {
		err = errors.New("marketplace package manifest not found")
		cleanup()
		return
	}

	manifestData, readErr := os.ReadFile(manifestPath)
	if readErr != nil {
		err = readErr
		cleanup()
		return
	}
	pkg, err = parseInstallPackageManifest(manifestData)
	if err != nil || pkg == nil {
		err = errors.New("invalid marketplace package manifest")
		cleanup()
	}
	return
}

func extractLocalPackageReaderContext(ctx context.Context, reader *zip.Reader, destination string) error {
	if err := ctx.Err(); err != nil {
		return err
	}
	if err := validateLocalPackageArchive(reader); err != nil {
		return err
	}
	if err := os.MkdirAll(destination, 0755); err != nil {
		return err
	}
	var extractedTotal uint64
	for _, item := range reader.File {
		if err := ctx.Err(); err != nil {
			return err
		}
		if err := extractLocalPackageItem(ctx, item, destination, &extractedTotal); err != nil {
			return err
		}
	}
	return ctx.Err()
}

func validateLocalPackageArchive(reader *zip.Reader) error {
	if len(reader.File) == 0 {
		return errors.New("marketplace package archive is empty")
	}
	if len(reader.File) > maxLocalPackageFileCount {
		return errors.New("marketplace package contains too many files")
	}

	var declaredTotal uint64
	seen := map[string]bool{}
	spellings := map[string]string{}
	for _, item := range reader.File {
		name := strings.TrimSuffix(item.Name, "/")
		if err := validateInstallRelativePath(name); err != nil {
			return err
		}
		key := strings.ToLower(name)
		if seen[key] {
			return errors.New("marketplace package contains duplicate or colliding paths")
		}
		seen[key] = true
		parts := strings.Split(name, "/")
		for i := range parts {
			prefix := strings.Join(parts[:i+1], "/")
			folded := strings.ToLower(prefix)
			if previous, ok := spellings[folded]; ok && previous != prefix {
				return errors.New("marketplace package contains colliding path components")
			}
			spellings[folded] = prefix
		}
		mode := item.Mode()
		if mode&os.ModeSymlink != 0 || (!mode.IsRegular() && !mode.IsDir()) {
			return errors.New("marketplace package contains an unsupported file")
		}
		if item.UncompressedSize64 > maxLocalPackageFileSize {
			return errors.New("marketplace package contains a file that is too large")
		}
		if ^uint64(0)-declaredTotal < item.UncompressedSize64 {
			return errors.New("marketplace package is too large")
		}
		declaredTotal += item.UncompressedSize64
		if declaredTotal > maxLocalPackageExtractSize {
			return errors.New("marketplace package is too large")
		}
	}

	return nil
}

func parseInstallPackageManifest(data []byte) (*Package, error) {
	var pkg *Package
	if err := gulu.JSON.UnmarshalJSON(data, &pkg); err != nil || pkg == nil {
		return nil, errors.New("invalid marketplace package manifest")
	}
	pkg.URL = strings.TrimSuffix(pkg.URL, "/")
	clearPackageDeprecationMetadata(pkg)
	return pkg, nil
}

// InspectLocalPackageWithHash 只在内存中查看同一份归档字节,不创建解压目录或修改安装目标。
func InspectLocalPackageWithHash(archivePath, expectedHash string) (pkgType string, pkg *Package, packageHash string, err error) {
	if expectedHash != "" && !validInstallHash(expectedHash) {
		return "", nil, "", errors.New("invalid package hash")
	}
	data, err := readInstallArchive(archivePath)
	if err != nil {
		return
	}
	packageHash = fmt.Sprintf("%x", sha256.Sum256(data))
	if expectedHash != "" && packageHash != expectedHash {
		err = ErrPackageHashConflict
		return
	}
	reader, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
	if err != nil {
		return
	}
	if err = validateLocalPackageArchive(reader); err != nil {
		return
	}
	tops := map[string]bool{}
	hasRootManifest := false
	for _, item := range reader.File {
		name := strings.TrimSuffix(item.Name, "/")
		tops[strings.Split(name, "/")[0]] = true
		if _, ok := localPackageManifests[name]; ok && item.Mode().IsRegular() {
			hasRootManifest = true
		}
	}
	prefix := ""
	if !hasRootManifest {
		if len(tops) != 1 {
			err = errors.New("marketplace package manifest must be at the archive root or its only top-level directory")
			return
		}
		for top := range tops {
			prefix = top + "/"
		}
	}
	var manifest *zip.File
	for _, item := range reader.File {
		name := strings.TrimPrefix(item.Name, prefix)
		if kind, ok := localPackageManifests[name]; ok && item.Mode().IsRegular() {
			if manifest != nil {
				err = errors.New("multiple marketplace package manifests found")
				return
			}
			manifest, pkgType = item, kind
		}
	}
	if manifest == nil {
		err = errors.New("marketplace package manifest not found")
		return
	}
	stream, err := manifest.Open()
	if err != nil {
		return
	}
	defer stream.Close()
	manifestData, err := io.ReadAll(io.LimitReader(stream, 1024*1024+1))
	if err != nil {
		return
	}
	if len(manifestData) > 1024*1024 {
		err = errors.New("marketplace package manifest is too large")
		return
	}
	pkg, err = parseInstallPackageManifest(manifestData)
	return
}

type localPackageContextReader struct {
	ctx    context.Context
	reader io.Reader
}

func (reader localPackageContextReader) Read(data []byte) (int, error) {
	if err := reader.ctx.Err(); err != nil {
		return 0, err
	}
	return reader.reader.Read(data)
}

func extractLocalPackageItem(ctx context.Context, item *zip.File, destination string, extractedTotal *uint64) error {
	name := strings.ReplaceAll(item.Name, "\\", "/")
	if name == "" || strings.HasPrefix(name, "/") {
		return errors.New("marketplace package contains an invalid path")
	}
	destinationPath := filepath.Join(destination, filepath.FromSlash(name))
	if !gulu.File.IsSubPath(destination, destinationPath) {
		return errors.New("marketplace package contains an invalid path")
	}

	mode := item.Mode()
	if mode&os.ModeSymlink != 0 || (!mode.IsRegular() && !mode.IsDir()) {
		return errors.New("marketplace package contains an unsupported file")
	}
	if mode.IsDir() {
		return os.MkdirAll(destinationPath, 0755)
	}
	if err := os.MkdirAll(filepath.Dir(destinationPath), 0755); err != nil {
		return err
	}

	source, err := item.Open()
	if err != nil {
		return err
	}
	defer source.Close()
	target, err := os.OpenFile(destinationPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
	if err != nil {
		return err
	}
	written, copyErr := io.Copy(target, io.LimitReader(localPackageContextReader{ctx: ctx, reader: source}, int64(maxLocalPackageFileSize)+1))
	closeErr := target.Close()
	if copyErr != nil {
		return copyErr
	}
	if closeErr != nil {
		return closeErr
	}
	if written > int64(maxLocalPackageFileSize) {
		return errors.New("marketplace package contains a file that is too large")
	}
	if uint64(written) > maxLocalPackageExtractSize-*extractedTotal {
		return errors.New("marketplace package is too large")
	}
	*extractedTotal += uint64(written)
	return nil
}

func localPackageRoot(extractPath string) (string, error) {
	if hasLocalPackageManifest(extractPath) {
		return extractPath, nil
	}
	entries, err := os.ReadDir(extractPath)
	if err != nil {
		return "", err
	}
	if len(entries) != 1 || !entries[0].IsDir() {
		return "", errors.New("marketplace package manifest must be at the archive root or its only top-level directory")
	}
	root := filepath.Join(extractPath, entries[0].Name())
	if !hasLocalPackageManifest(root) {
		return "", errors.New("marketplace package manifest not found")
	}
	return root, nil
}

func hasLocalPackageManifest(root string) bool {
	for manifestName := range localPackageManifests {
		if info, err := os.Stat(filepath.Join(root, manifestName)); err == nil && info.Mode().IsRegular() {
			return true
		}
	}
	return false
}