Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: add encrypt doc && refactoring encrypt code (#893)

baojun-zhang5bbb22a6ef88
diff --git a/docs/en/concepts/10-encryption.md b/docs/en/concepts/10-encryption.md
new file mode 100644
index 000000000..a00ef3122
--- /dev/null
+++ b/docs/en/concepts/10-encryption.md
@@ -0,0 +1,220 @@
+# Data Encryption
+
+OpenViking provides transparent at-rest data encryption to ensure data security and isolation in multi-tenant environments.
+
+## Overview
+
+### Why Encryption
+
+In a multi-tenant architecture, resources, memories, and skills from different customers (accounts) are stored in a shared AGFS instance. Encryption ensures:
+
+- Even if an attacker gains AGFS disk access, they cannot read any customer's plaintext data
+- Different accounts' data is encrypted with independent keys for tenant isolation
+- All encryption/decryption operations are centralized at the VikingFS layer; AGFS and external object stores only see ciphertext
+
+### Transparency
+
+Encryption is completely transparent to users and developers:
+
+- **No client API changes**: Existing code works without modification
+- **Application layer unaware**: Read/write operations behave exactly like unencrypted
+- **Backward compatible**: Unencrypted old files can still be read normally
+
+## Three-Layer Key Architecture
+
+OpenViking uses an Envelope Encryption architecture with a three-layer key system:
+
+```
+┌─────────────────────────────────────────────────────────┐
+│ Layer 1: Root Key │
+│ • Global unique per OpenViking instance │
+│ • Storage: KMS service / ~/.openviking/master.key │
+│ • Purpose: Derive all account keys │
+└────────────────────┬────────────────────────────────────┘
+ │ HKDF derivation
+ ▼
+┌─────────────────────────────────────────────────────────┐
+│ Layer 2: Account Key (KEK) │
+│ • One independent key per account │
+│ • Not stored, derived at runtime │
+│ • Purpose: Encrypt all file keys for this account │
+└────────────────────┬────────────────────────────────────┘
+ │ AES-256-GCM encryption
+ ▼
+┌─────────────────────────────────────────────────────────┐
+│ Layer 3: File Key (DEK) │
+│ • New random key generated per write operation │
+│ • Stored encrypted in file header (envelope) │
+│ • Purpose: Encrypt actual file content │
+└─────────────────────────────────────────────────────────┘
+```
+
+### Key Hierarchy Summary
+
+| Layer | Name | Description | Quantity |
+|-------|------|-------------|----------|
+| **Root Key** | Root Key | System master key, used to derive all account keys | 1 per instance |
+| **Account Key** | Account Key | Independent key per account, derived from root key | 1 per account |
+| **File Key** | File Key | One-time random key per file | 1 per write |
+
+## Key Providers
+
+OpenViking supports three key providers for different deployment scenarios:
+
+| Provider | Use Case | Root Key Storage | Features |
+|----------|----------|-----------------|----------|
+| **Local** | Dev environments, single-node deployments | Local file `~/.openviking/master.key` | Simple, no external services |
+| **Vault** | Production, multi-cloud | HashiCorp Vault Transit Engine | Enterprise-grade key management, version control |
+| **Volcengine KMS** | Volcengine cloud deployments | Volcengine KMS | Cloud-native KMS service |
+
+### Local (File)
+
+Suitable for development and single-node deployments:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ }
+ }
+}
+```
+
+**Initialization command**:
+```bash
+ov crypto init-key --output ~/.openviking/master.key
+```
+
+### Vault (HashiCorp Vault)
+
+Suitable for production and multi-cloud deployments:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "vault",
+ "vault": {
+ "address": "https://vault.example.com:8200",
+ "token": "hvs.your-vault-token",
+ "mount_point": "transit",
+ "kv_mount_point": "secret",
+ "kv_version": 1,
+ "root_key_name": "openviking-root-key",
+ "encrypted_root_key_key": "openviking-encrypted-root-key"
+ }
+ }
+}
+```
+
+### Volcengine KMS
+
+Suitable for Volcengine cloud deployments:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "volcengine_kms",
+ "volcengine_kms": {
+ "key_id": "d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+ "region": "cn-beijing",
+ "access_key": "AKLTxxxxxxxxxxxxxxxxxx",
+ "secret_key": "Tmpxxxxxxxxxxxxxxxxxxxxxx",
+ "endpoint": null,
+ "key_file": "~/.openviking/openviking-volcengine-root-key.enc"
+ }
+ }
+}
+```
+
+## How It Works
+
+### Write Flow
+
+```
+Client VikingFS FileEncryptor KeyManager AGFS
+ │ │ │ │ │
+ │ write(uri, data) │ │ │ │
+ │──────────────────>│ │ │ │
+ │ │ encrypt(account_id, │ │ │
+ │ │ plaintext) │ │ │
+ │ │──────────────────────>│ │ │
+ │ │ │ derive_account_key()│ │
+ │ │ │────────────────────>│ │
+ │ │ │<────────────────────│ │
+ │ │ │ account_key │ │
+ │ │ 1. Generate random File Key │
+ │ │ 2. Encrypt content with File Key │
+ │ │ 3. Encrypt File Key with Account Key │
+ │ │ 4. Build envelope format │
+ │ │<──────────────────────│ │ │
+ │ │ ciphertext │ │ │
+ │ │──────────────────────────────────────────────────────────>│
+ │ │ │ │ Write │
+ │<──────────────────│ │ │ │
+ │ success │ │ │ │
+```
+
+### Read Flow
+
+```
+Client VikingFS FileEncryptor KeyManager AGFS
+ │ │ │ │ │
+ │ read(uri) │ │ │ │
+ │──────────────────>│ │ │ │
+ │ │──────────────────────────────────────────────────────────>│
+ │ │ │ │ Read │
+ │ │<──────────────────────────────────────────────────────────│
+ │ │ raw_bytes │ │ │
+ │ │ Check magic == "OVE1"?│ │ │
+ │ │ Yes → decrypt() │ │ │
+ │ │──────────────────────>│ │ │
+ │ │ │ derive_account_key()│ │
+ │ │ │────────────────────>│ │
+ │ │ │<────────────────────│ │
+ │ │ │ account_key │ │
+ │ │ 1. Parse envelope format │
+ │ │ 2. Decrypt File Key with Account Key │
+ │ │ 3. Decrypt content with File Key │
+ │ │<──────────────────────│ │ │
+ │ │ plaintext │ │ │
+ │<──────────────────│ │ │ │
+ │ content │ │ │ │
+```
+
+### Envelope Format
+
+Encrypted files use a unified envelope format starting with the magic number `OVE1` (OpenViking Encryption v1):
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│ Magic │ Version │ Provider │ Encrypted File Key │ ... │
+│ 4 bytes │ 1 byte │ 1 byte │ Variable length │ ... │
+│ "OVE1" │ 0x01 │ 0x01=local│ │ ... │
+└─────────────────────────────────────────────────────────────┘
+```
+
+- If a file doesn't start with `OVE1`, it's treated as unencrypted and plaintext is returned directly
+- Backward compatible, old files don't need migration
+
+## Multi-Tenant Isolation
+
+Different accounts' data is encrypted with independent Account Keys:
+
+- Account A's key cannot decrypt Account B's files
+- Even with full AGFS access, data can't be read without the corresponding key
+- Tenant isolation is implemented at the key layer, not relying on storage permissions
+
+## Configuration Example
+
+See [Configuration Guide](../guides/01-configuration.md#encryption) for detailed configuration.
+
+## Related Documentation
+
+- [Storage Architecture](./05-storage.md) - VikingFS and AGFS architecture
+- [Configuration Guide](../guides/01-configuration.md) - Encryption configuration details
+- [Technical Design](../../design/multi-tenant-file-encryption-desigin.md) - Encryption technical design
diff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.md
index 48003e28e..576ed8566 100644
--- a/docs/en/guides/01-configuration.md
+++ b/docs/en/guides/01-configuration.md
@@ -786,6 +786,98 @@ Path locks are enabled by default and usually require no configuration. **The de
For details on the lock mechanism, see [Path Locks and Crash Recovery](../concepts/09-transaction.md).
+## encryption Section
+
+Enable at-rest data encryption to ensure data security and isolation in multi-tenant environments. Encryption is completely transparent to users with no API changes.
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local|vault|volcengine_kms"
+ }
+}
+```
+
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `enabled` | bool | Whether encryption is enabled | `false` |
+| `provider` | str | Key provider: `"local"`, `"vault"`, or `"volcengine_kms"` | - |
+
+### Local (File)
+
+Suitable for development environments and single-node deployments:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ }
+ }
+}
+```
+
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `local.key_file` | str | Root key file path | `~/.openviking/master.key` |
+
+### Vault (HashiCorp Vault)
+
+Suitable for production and multi-cloud deployments:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "vault",
+ "vault": {
+ "address": "https://vault.example.com:8200",
+ "token": "vault-token-xxx",
+ "mount_point": "transit",
+ "key_name": "openviking-root"
+ }
+ }
+}
+```
+
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `vault.address` | str | Vault service address | - |
+| `vault.token` | str | Vault access token | - |
+| `vault.mount_point` | str | Transit engine mount point | `"transit"` |
+| `vault.key_name` | str | Root key name | `"openviking-root"` |
+
+### Volcengine KMS
+
+Suitable for Volcengine cloud deployments:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "volcengine_kms",
+ "volcengine_kms": {
+ "key_id": "kms-key-id-xxx",
+ "region": "cn-beijing",
+ "access_key": "AKLTxxxxxxxx",
+ "secret_key": "Tmpxxxxxxxx"
+ }
+ }
+}
+```
+
+| Parameter | Type | Description | Default |
+|-----------|------|-------------|---------|
+| `volcengine_kms.key_id` | str | KMS key ID | - |
+| `volcengine_kms.region` | str | Region | `"cn-beijing"` |
+| `volcengine_kms.access_key` | str | Volcengine Access Key | - |
+| `volcengine_kms.secret_key` | str | Volcengine Secret Key | - |
+
+For detailed encryption explanations, see [Data Encryption](../concepts/10-encryption.md). For complete usage instructions, see [Encryption Guide](./08-encryption.md).
+
## Full Schema
```json
@@ -817,6 +909,25 @@ For details on the lock mechanism, see [Path Locks and Crash Recovery](../concep
"api_base": "string",
"threshold": 0.1
},
+ "encryption": {
+ "enabled": false,
+ "provider": "local|vault|volcengine_kms",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ },
+ "vault": {
+ "address": "https://vault.example.com:8200",
+ "token": "string",
+ "mount_point": "transit",
+ "key_name": "openviking-root"
+ },
+ "volcengine_kms": {
+ "key_id": "string",
+ "region": "cn-beijing",
+ "access_key": "string",
+ "secret_key": "string"
+ }
+ },
"storage": {
"workspace": "string",
"agfs": {
diff --git a/docs/en/guides/08-encryption.md b/docs/en/guides/08-encryption.md
new file mode 100644
index 000000000..e7aaf297d
--- /dev/null
+++ b/docs/en/guides/08-encryption.md
@@ -0,0 +1,401 @@
+# Encryption Guide
+
+This guide describes how to enable and use at-rest data encryption in OpenViking.
+
+## Overview
+
+OpenViking provides transparent at-rest data encryption to ensure data security and isolation in multi-tenant environments:
+
+- ✅ **Transparent encryption**: No API changes, application layer unaware
+- ✅ **Multi-tenant isolation**: Different accounts use independent keys
+- ✅ **Three key providers**: Local, Vault, Volcengine KMS
+- ✅ **Backward compatible**: Unencrypted old files still readable
+
+See [Data Encryption](../concepts/10-encryption.md) for conceptual explanations.
+
+## Quick Start
+
+### 1. Initialize Root Key (Local Mode)
+
+```bash
+ov system crypto init-key --output ~/.openviking/master.key
+```
+
+### 2. Configure Encryption
+
+Edit `~/.openviking/ov.conf`:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ }
+ },
+ "storage": {
+ "workspace": "./data"
+ }
+}
+```
+
+### 3. Verify
+
+```python
+import openviking as ov
+import asyncio
+
+
+async def test():
+ client = ov.AsyncOpenViking(path="./data")
+ await client.initialize()
+
+ # Add resource (automatically encrypted)
+ await client.add_resource("Hello, encrypted world!", reason="Test encryption")
+
+ # Read resource (automatically decrypted)
+ results = await client.find("encrypted")
+ print(f"Found {len(results)} results")
+
+ await client.close()
+
+
+asyncio.run(test())
+```
+
+Done! Now all written data is automatically encrypted.
+
+## Choosing a Key Provider
+
+| Provider | Use Case | Pros | Cons |
+|----------|----------|------|------|
+| **Local** | Dev environments, single-node | Simple, no external services | Key stored locally, less secure |
+| **Vault** | Production, multi-cloud | Enterprise-grade KMS, version control | Requires deploying and maintaining Vault |
+| **Volcengine KMS** | Volcengine cloud | Cloud-native KMS service | Volcengine-only |
+
+---
+
+## Local Mode Detailed Guide
+
+### Initialize Root Key
+
+```bash
+# Generate and save to specified path
+ov system crypto init-key --output ~/.openviking/master.key
+
+# Or use short option
+ov system crypto init-key -o ~/.openviking/master.key
+```
+
+**Output example**:
+```
+✓ Root key generated successfully
+✓ Saved to: /Users/you/.openviking/master.key
+```
+
+### Security Tips
+
+- ⚠️ Keep `master.key` safe
+- Recommend setting file permissions to `600` (owner-only read/write)
+- Regularly back up the key file
+- Don't commit the key file to version control
+
+### Configuration Example
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ }
+ }
+}
+```
+
+---
+
+## Vault Mode Detailed Guide
+
+### Prerequisites
+
+1. HashiCorp Vault service deployed
+2. Transit engine enabled
+3. Vault Token with sufficient permissions
+
+### Configure Vault
+
+1. Enable Transit engine (if not already enabled):
+
+```bash
+vault secrets enable transit
+```
+
+2. Enable KV engine (if not already enabled):
+
+```bash
+# KV v2 (recommended)
+vault secrets enable -version=2 kv
+
+# Or KV v1
+vault secrets enable kv
+```
+
+3. Configure OpenViking:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "vault",
+ "vault": {
+ "address": "https://vault.example.com:8200",
+ "token": "hvs.xxxxxxxxxxxxxxxxxxxxx",
+ "mount_point": "transit",
+ "kv_mount_point": "secret",
+ "kv_version": 1,
+ "root_key_name": "openviking-root-key",
+ "encrypted_root_key_key": "openviking-encrypted-root-key"
+ }
+ }
+}
+```
+
+**Configuration Parameters**:
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `address` | Vault server address | Required |
+| `token` | Vault authentication token | Required |
+| `mount_point` | Transit engine mount path | `"transit"` |
+| `kv_mount_point` | KV engine mount path | `"secret"` |
+| `kv_version` | KV engine version (1 or 2) | `1` |
+| `root_key_name` | Key name in Transit engine | `"openviking-root-key"` |
+| `encrypted_root_key_key` | Path to store encrypted root key in KV engine | `"openviking-encrypted-root-key"` |
+
+### Vault Permission Recommendations
+
+Configure minimal permissions for the Token:
+
+```hcl
+path "transit/encrypt/openviking-root" {
+ capabilities = ["update"]
+}
+
+path "transit/decrypt/openviking-root" {
+ capabilities = ["update"]
+}
+```
+
+---
+
+## Volcengine KMS Mode Detailed Guide
+
+### Prerequisites
+
+1. Volcengine KMS service activated
+2. Symmetric key created
+3. Valid Access Key and Secret Key
+
+### Create KMS Key
+
+1. Visit [Volcengine KMS Console](https://console.volcengine.com/kms)
+2. Click "Create Key"
+3. Select "Symmetric Key", algorithm `AES_256`
+4. Record the Key ID
+
+### Configure OpenViking
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "volcengine_kms",
+ "volcengine_kms": {
+ "key_id": "d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+ "region": "cn-beijing",
+ "access_key": "AKLTxxxxxxxxxxxxxxxxxx",
+ "secret_key": "Tmpxxxxxxxxxxxxxxxxxxxxxx",
+ "endpoint": null,
+ "key_file": "~/.openviking/openviking-volcengine-root-key.enc"
+ }
+ }
+}
+```
+
+**Configuration Parameters**:
+
+| Parameter | Description | Default |
+|-----------|-------------|---------|
+| `key_id` | KMS Key ID | Required |
+| `region` | Region | Required |
+| `access_key` | Access Key | Required |
+| `secret_key` | Secret Key | Required |
+| `endpoint` | Custom KMS endpoint (optional) | `null` (use default endpoint) |
+| `key_file` | Local cache file path for encrypted root key | `"~/.openviking/openviking-volcengine-root-key.enc"` |
+
+### Permission Recommendations
+
+Configure minimal permissions for the Access Key:
+
+```json
+{
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "kms:Encrypt",
+ "kms:Decrypt"
+ ],
+ "Resource": [
+ "trn:kms:*:*:key/d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+ ]
+ }
+ ]
+}
+```
+
+---
+
+## Verifying Encryption
+
+### Method 1: Check File Content
+
+Encrypted files start with magic number `OVE1`:
+
+```bash
+# View first 4 bytes
+hexdump -C ./data/agfs/your-file | head -1
+```
+
+**Encrypted file**:
+```
+00000000 4f 56 45 31 01 01 00 00 00 20 8a 7b 2c 9d 1e |OVE1..... .{,..|
+```
+(First 4 bytes are `4f 56 45 31` = "OVE1")
+
+**Unencrypted file**:
+```
+00000000 7b 22 63 6f 6e 74 65 6e 74 73 22 3a 5b 7b 22 70 |{"contents":[{"p|
+```
+
+### Method 2: Cross-Provider Verification
+
+Try decrypting with different providers — it should fail (this is normal security behavior):
+
+```python
+# Encrypt with Provider A
+encrypted = await provider_a.encrypt_file_key(plaintext, "test-account")
+
+# Try decrypting with Provider B (should fail)
+try:
+ await provider_b.decrypt_file_key(encrypted, "test-account")
+ print("❌ Security vulnerability: Cross-provider decryption succeeded!")
+except Exception as e:
+ print("✓ Secure: Cross-provider decryption failed as expected")
+```
+
+---
+
+## Migration Guide
+
+### Migrating from Unencrypted to Encrypted
+
+1. Back up existing data
+2. Enable encryption (see above)
+3. Re-import all resources:
+
+```python
+import openviking as ov
+import asyncio
+
+
+async def migrate():
+ client = ov.AsyncOpenViking(path="./data")
+ await client.initialize()
+
+ # List all resources
+ resources = await client.list_resources()
+
+ for resource in resources:
+ # Read old resource (unencrypted)
+ content = await client.read_resource(resource["uri"])
+ # Re-write (automatically encrypted)
+ await client.add_resource(content, reason="Migrate to encrypted storage")
+
+ await client.close()
+
+
+asyncio.run(migrate())
+```
+
+### Switching Key Providers
+
+1. Back up existing data and keys
+2. Decrypt all data with old provider
+3. Configure new provider
+4. Re-encrypt all data
+
+**Note**: This is a destructive operation, recommend testing first in a staging environment.
+
+---
+
+## Troubleshooting
+
+### Key File Not Found
+
+```
+Error: Key file not found: ~/.openviking/master.key
+```
+
+**Solution**:
+1. Check file path is correct
+2. Use absolute path
+3. Ensure `~` is properly expanded (use `expanduser()`)
+
+### Vault Connection Failed
+
+```
+Error: Failed to connect to Vault
+```
+
+**Solution**:
+1. Check if Vault service is running
+2. Verify `address` configuration
+3. Check network connectivity and firewall
+4. Confirm Token is valid and not expired
+
+### Volcengine KMS Authentication Failed
+
+```
+Error: Invalid credentials
+```
+
+**Solution**:
+1. Check Access Key and Secret Key are correct
+2. Confirm key has sufficient permissions
+3. Verify region configuration is correct
+
+### Cross-Provider Decryption Failed (This is Normal)
+
+```
+Error: KeyMismatchError
+```
+
+**Explanation**: This is expected security behavior. Different providers use different root keys and cannot decrypt each other's data.
+
+### Partial Read Returns Ciphertext
+
+If using encrypted files created with an older OpenViking version, partial reads may return ciphertext.
+
+**Solution**: Upgrade to the latest OpenViking version.
+
+---
+
+## Related Documentation
+
+- [Data Encryption](../concepts/10-encryption.md) - Encryption concepts
+- [Configuration Guide](./01-configuration.md) - Complete configuration reference
+- [Technical Design](../../design/multi-tenant-file-encryption-desigin.md) - Encryption technical design
diff --git a/docs/zh/concepts/10-encryption.md b/docs/zh/concepts/10-encryption.md
new file mode 100644
index 000000000..2c1b583ae
--- /dev/null
+++ b/docs/zh/concepts/10-encryption.md
@@ -0,0 +1,220 @@
+# 数据加密
+
+OpenViking 提供透明的静态数据加密,确保多租户环境下的数据安全与隔离。
+
+## 概述
+
+### 为什么需要加密
+
+在多租户架构中,不同客户(账户)的资源文件、记忆和技能都存储在共享的 AGFS 实例中。加密确保:
+
+- 即使攻击者获得 AGFS 磁盘访问权限,也无法读取任何客户的明文数据
+- 不同账户的数据使用独立密钥加密,实现租户隔离
+- 所有加密解密操作集中在 VikingFS 层,AGFS 和外部对象存储只看到密文
+
+### 对谁透明
+
+加密功能对用户和开发者完全透明:
+
+- **客户端 API 无变化**:现有代码无需修改
+- **应用层无感知**:读写操作与未加密时完全相同
+- **向后兼容**:未加密的旧文件仍可正常读取
+
+## 三层密钥架构
+
+OpenViking 采用信封加密(Envelope Encryption)架构,使用三层密钥体系:
+
+```
+┌─────────────────────────────────────────────────────────┐
+│ Layer 1: Root Key(根密钥) │
+│ • 整个 OpenViking 实例全局唯一 │
+│ • 存储:KMS 服务 / ~/.openviking/master.key │
+│ • 用途:派生所有账户密钥 │
+└────────────────────┬────────────────────────────────────┘
+ │ HKDF 派生
+ ▼
+┌─────────────────────────────────────────────────────────┐
+│ Layer 2: Account Key(账户密钥,KEK) │
+│ • 每个账户一个独立密钥 │
+│ • 不存储,运行时派生 │
+│ • 用途:加密该账户下的所有文件密钥 │
+└────────────────────┬────────────────────────────────────┘
+ │ AES-256-GCM 加密
+ ▼
+┌─────────────────────────────────────────────────────────┐
+│ Layer 3: File Key(文件密钥,DEK) │
+│ • 每次写操作生成新的随机密钥 │
+│ • 加密后存储在文件头(信封)中 │
+│ • 用途:加密实际文件内容 │
+└─────────────────────────────────────────────────────────┘
+```
+
+### 密钥层次说明
+
+| 层级 | 名称 | 说明 | 数量 |
+|------|------|------|------|
+| **Root Key** | 根密钥 | 整个系统的主密钥,用于派生所有账户密钥 | 1 个实例 |
+| **Account Key** | 账户密钥 | 每个账户独立的密钥,从根密钥派生 | 每个账户 1 个 |
+| **File Key** | 文件密钥 | 每个文件的一次性随机密钥 | 每次写入 1 个 |
+
+## 密钥提供程序
+
+OpenViking 支持三种密钥提供程序,适应不同的部署场景:
+
+| 提供程序 | 适用场景 | Root Key 存储 | 特点 |
+|---------|---------|--------------|------|
+| **Local** | 开发环境、单节点部署 | 本地文件 `~/.openviking/master.key` | 简单,无需外部服务 |
+| **Vault** | 生产环境、多云部署 | HashiCorp Vault Transit Engine | 企业级密钥管理,支持密钥版本控制 |
+| **Volcengine KMS** | 火山引擎云部署 | 火山引擎 KMS | 云原生密钥管理服务 |
+
+### Local(本地文件)
+
+适合开发环境和单节点部署:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ }
+ }
+}
+```
+
+**初始化命令**:
+```bash
+ov crypto init-key --output ~/.openviking/master.key
+```
+
+### Vault(HashiCorp Vault)
+
+适合生产环境和多云部署:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "vault",
+ "vault": {
+ "address": "https://vault.example.com:8200",
+ "token": "hvs.your-vault-token",
+ "mount_point": "transit",
+ "kv_mount_point": "secret",
+ "kv_version": 1,
+ "root_key_name": "openviking-root-key",
+ "encrypted_root_key_key": "openviking-encrypted-root-key"
+ }
+ }
+}
+```
+
+### Volcengine KMS(火山引擎)
+
+适合火山引擎云部署:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "volcengine_kms",
+ "volcengine_kms": {
+ "key_id": "d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+ "region": "cn-beijing",
+ "access_key": "AKLTxxxxxxxxxxxxxxxxxx",
+ "secret_key": "Tmpxxxxxxxxxxxxxxxxxxxxxx",
+ "endpoint": null,
+ "key_file": "~/.openviking/openviking-volcengine-root-key.enc"
+ }
+ }
+}
+```
+
+## 工作原理
+
+### 写流程
+
+```
+客户端 VikingFS FileEncryptor KeyManager AGFS
+ │ │ │ │ │
+ │ write(uri, data) │ │ │ │
+ │──────────────────>│ │ │ │
+ │ │ encrypt(account_id, │ │ │
+ │ │ plaintext) │ │ │
+ │ │──────────────────────>│ │ │
+ │ │ │derive_account_key()│ │
+ │ │ │───────────────────>│ │
+ │ │ │<───────────────────│ │
+ │ │ │ account_key │ │
+ │ │ 1. 生成随机 File Key │ │
+ │ │ 2. 用 File Key 加密内容 │ │
+ │ │ 3. 用 Account Key 加密 File Key │ │
+ │ │ 4. 构建信封格式 │ │
+ │ │<──────────────────────│ │ │
+ │ │ ciphertext │ │ │
+ │ │─────────────────────────────────────────────────────────>│
+ │ │ │ │ Write │
+ │<──────────────────│ │ │ │
+ │ success │ │ │ │
+```
+
+### 读流程
+
+```
+客户端 VikingFS FileEncryptor KeyManager AGFS
+ │ │ │ │ │
+ │ read(uri) │ │ │ │
+ │──────────────────>│ │ │ │
+ │ │──────────────────────────────────────────────────────────>│
+ │ │ │ │ Read │
+ │ │<──────────────────────────────────────────────────────────│
+ │ │ raw_bytes │ │ │
+ │ │ 检查魔术数 == "OVE1"? │ │ │
+ │ │ 是 → decrypt() │ │ │
+ │ │──────────────────────>│ │ │
+ │ │ │ derive_account_key()│ │
+ │ │ │────────────────────>│ │
+ │ │ │<────────────────────│ │
+ │ │ │ account_key │ │
+ │ │ 1. 解析信封格式 │ │
+ │ │ 2. 用 Account Key 解密 File Key │ │
+ │ │ 3. 用 File Key 解密内容 │ │
+ │ │<──────────────────────│ │ │
+ │ │ plaintext │ │ │
+ │<──────────────────│ │ │ │
+ │ content │ │ │ │
+```
+
+### 信封格式
+
+加密文件使用统一的信封格式,以魔术数 `OVE1`(OpenViking Encryption v1)开头:
+
+```
+┌─────────────────────────────────────────────────────────────┐
+│ 魔术数 │ 版本 │ Provider │ 加密的 File Key │ .. │
+│ 4 字节 │ 1 字节 │ 1 字节 │ 可变长度 │ ... │
+│ "OVE1" │ 0x01 │ 0x01=local │ │ ... │
+└─────────────────────────────────────────────────────────────┘
+```
+
+- 如果文件不以 `OVE1` 开头,视为未加密文件,直接返回明文
+- 支持向后兼容,旧文件无需迁移
+
+## 多租户隔离
+
+不同账户的数据使用独立的 Account Key 加密:
+
+- 账户 A 的密钥无法解密账户 B 的文件
+- 即使 AGFS 被完全访问,没有对应密钥也无法读取数据
+- 租户隔离在密钥层面实现,不依赖存储层权限
+
+## 配置示例
+
+详细配置说明请参考 [配置文档](../guides/01-configuration.md#encryption)。
+
+## 相关文档
+
+- [存储架构](./05-storage.md) - VikingFS 和 AGFS 架构
+- [配置指南](../guides/01-configuration.md) - 加密配置详解
+- [技术设计](../../design/multi-tenant-file-encryption-desigin.md) - 加密技术设计文档
diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.md
index a5c3507df..83d05327c 100644
--- a/docs/zh/guides/01-configuration.md
+++ b/docs/zh/guides/01-configuration.md
@@ -741,6 +741,98 @@ HTTP 客户端(`SyncHTTPClient` / `AsyncHTTPClient`)和 CLI 工具连接远
启动方式和部署详情见 [服务部署](./03-deployment.md),认证详情见 [认证](./04-authentication.md)。
+## encryption 段
+
+启用静态数据加密,确保多租户环境下的数据安全与隔离。加密功能对用户完全透明,API 无变化。
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local|vault|volcengine_kms"
+ }
+}
+```
+
+| 参数 | 类型 | 说明 | 默认值 |
+|------|------|------|--------|
+| `enabled` | bool | 是否启用加密 | `false` |
+| `provider` | str | 密钥提供程序:`"local"`、`"vault"` 或 `"volcengine_kms"` | - |
+
+### Local(本地文件)
+
+适合开发环境和单节点部署:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ }
+ }
+}
+```
+
+| 参数 | 类型 | 说明 | 默认值 |
+|------|------|------|--------|
+| `local.key_file` | str | 根密钥文件路径 | `~/.openviking/master.key` |
+
+### Vault(HashiCorp Vault)
+
+适合生产环境和多云部署:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "vault",
+ "vault": {
+ "address": "https://vault.example.com:8200",
+ "token": "vault-token-xxx",
+ "mount_point": "transit",
+ "key_name": "openviking-root"
+ }
+ }
+}
+```
+
+| 参数 | 类型 | 说明 | 默认值 |
+|------|------|------|--------|
+| `vault.address` | str | Vault 服务地址 | - |
+| `vault.token` | str | Vault 访问令牌 | - |
+| `vault.mount_point` | str | Transit 引擎挂载点 | `"transit"` |
+| `vault.key_name` | str | 根密钥名称 | `"openviking-root"` |
+
+### Volcengine KMS(火山引擎)
+
+适合火山引擎云部署:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "volcengine_kms",
+ "volcengine_kms": {
+ "key_id": "kms-key-id-xxx",
+ "region": "cn-beijing",
+ "access_key": "AKLTxxxxxxxx",
+ "secret_key": "Tmpxxxxxxxx"
+ }
+ }
+}
+```
+
+| 参数 | 类型 | 说明 | 默认值 |
+|------|------|------|--------|
+| `volcengine_kms.key_id` | str | KMS 密钥 ID | - |
+| `volcengine_kms.region` | str | 区域 | `"cn-beijing"` |
+| `volcengine_kms.access_key` | str | 火山引擎 Access Key | - |
+| `volcengine_kms.secret_key` | str | 火山引擎 Secret Key | - |
+
+加密功能的详细说明见 [数据加密](../concepts/10-encryption.md),完整使用流程见 [加密指南](./08-encryption.md)。
+
## storage.transaction 段
路径锁默认启用,通常无需配置。**默认行为是不等待**:若目标路径已被其他操作锁定,操作立即失败并抛出 `LockAcquisitionError`。若需要等待重试,请将 `lock_timeout` 设为正数。
@@ -794,6 +886,25 @@ HTTP 客户端(`SyncHTTPClient` / `AsyncHTTPClient`)和 CLI 工具连接远
"api_base": "string",
"threshold": 0.1
},
+ "encryption": {
+ "enabled": false,
+ "provider": "local|vault|volcengine_kms",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ },
+ "vault": {
+ "address": "https://vault.example.com:8200",
+ "token": "string",
+ "mount_point": "transit",
+ "key_name": "openviking-root"
+ },
+ "volcengine_kms": {
+ "key_id": "string",
+ "region": "cn-beijing",
+ "access_key": "string",
+ "secret_key": "string"
+ }
+ },
"storage": {
"workspace": "string",
"agfs": {
diff --git a/docs/zh/guides/08-encryption.md b/docs/zh/guides/08-encryption.md
new file mode 100644
index 000000000..9e8d60fb5
--- /dev/null
+++ b/docs/zh/guides/08-encryption.md
@@ -0,0 +1,401 @@
+# 加密指南
+
+本指南介绍如何在 OpenViking 中启用和使用静态数据加密功能。
+
+## 概述
+
+OpenViking 提供透明的静态数据加密,确保多租户环境下的数据安全与隔离:
+
+- ✅ **透明加密**:API 无变化,应用层无感知
+- ✅ **多租户隔离**:不同账户使用独立密钥
+- ✅ **三种密钥提供程序**:Local、Vault、火山引擎 KMS
+- ✅ **向后兼容**:未加密的旧文件仍可正常读取
+
+加密功能的概念说明见 [数据加密](../concepts/10-encryption.md)。
+
+## 快速开始
+
+### 1. 初始化根密钥(Local 模式)
+
+```bash
+ov system crypto init-key --output ~/.openviking/master.key
+```
+
+### 2. 配置加密
+
+编辑 `~/.openviking/ov.conf`:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ }
+ },
+ "storage": {
+ "workspace": "./data"
+ }
+}
+```
+
+### 3. 验证
+
+```python
+import openviking as ov
+import asyncio
+
+
+async def test():
+ client = ov.AsyncOpenViking(path="./data")
+ await client.initialize()
+
+ # 添加资源(自动加密)
+ await client.add_resource("Hello, encrypted world!", reason="测试加密")
+
+ # 读取资源(自动解密)
+ results = await client.find("encrypted")
+ print(f"找到 {len(results)} 个结果")
+
+ await client.close()
+
+
+asyncio.run(test())
+```
+
+完成!现在所有写入的数据都会自动加密。
+
+## 密钥提供程序选择
+
+| 提供程序 | 适用场景 | 优点 | 缺点 |
+|---------|---------|------|------|
+| **Local** | 开发环境、单节点部署 | 简单,无需外部服务 | 密钥存储在本地,安全性较低 |
+| **Vault** | 生产环境、多云部署 | 企业级密钥管理,支持版本控制 | 需要部署和维护 Vault |
+| **Volcengine KMS** | 火山引擎云部署 | 云原生密钥管理服务 | 仅限火山引擎环境 |
+
+---
+
+## Local 模式详细指南
+
+### 初始化根密钥
+
+```bash
+# 生成并保存到指定路径
+ov system crypto init-key --output ~/.openviking/master.key
+
+# 或者使用简短命令
+ov system crypto init-key -o ~/.openviking/master.key
+```
+
+**输出示例**:
+```
+✓ Root key generated successfully
+✓ Saved to: /Users/you/.openviking/master.key
+```
+
+### 安全提示
+
+- ⚠️ 妥善保管 `master.key` 文件
+- 建议设置文件权限为 `600`(仅所有者可读写)
+- 定期备份密钥文件
+- 不要将密钥文件提交到版本控制系统
+
+### 配置示例
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "local",
+ "local": {
+ "key_file": "~/.openviking/master.key"
+ }
+ }
+}
+```
+
+---
+
+## Vault 模式详细指南
+
+### 前置条件
+
+1. 已部署 HashiCorp Vault 服务
+2. 已启用 Transit 引擎
+3. 有足够权限的 Vault Token
+
+### 配置 Vault
+
+1. 启用 Transit 引擎(如果尚未启用):
+
+```bash
+vault secrets enable transit
+```
+
+2. 启用 KV 引擎(如果尚未启用):
+
+```bash
+# KV v2(推荐)
+vault secrets enable -version=2 kv
+
+# 或 KV v1
+vault secrets enable kv
+```
+
+3. 配置 OpenViking:
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "vault",
+ "vault": {
+ "address": "https://vault.example.com:8200",
+ "token": "hvs.xxxxxxxxxxxxxxxxxxxxx",
+ "mount_point": "transit",
+ "kv_mount_point": "secret",
+ "kv_version": 1,
+ "root_key_name": "openviking-root-key",
+ "encrypted_root_key_key": "openviking-encrypted-root-key"
+ }
+ }
+}
+```
+
+**配置参数说明**:
+
+| 参数 | 说明 | 默认值 |
+|------|------|--------|
+| `address` | Vault 服务器地址 | 必需 |
+| `token` | Vault 认证令牌 | 必需 |
+| `mount_point` | Transit 引擎挂载路径 | `"transit"` |
+| `kv_mount_point` | KV 引擎挂载路径 | `"secret"` |
+| `kv_version` | KV 引擎版本(1 或 2) | `1` |
+| `root_key_name` | Transit 引擎中的密钥名称 | `"openviking-root-key"` |
+| `encrypted_root_key_key` | KV 引擎中存储加密根密钥的路径 | `"openviking-encrypted-root-key"` |
+
+### Vault 权限建议
+
+为 Token 配置最小权限:
+
+```hcl
+path "transit/encrypt/openviking-root" {
+ capabilities = ["update"]
+}
+
+path "transit/decrypt/openviking-root" {
+ capabilities = ["update"]
+}
+```
+
+---
+
+## Volcengine KMS 模式详细指南
+
+### 前置条件
+
+1. 已开通火山引擎 KMS 服务
+2. 已创建对称密钥
+3. 有有效的 Access Key 和 Secret Key
+
+### 创建 KMS 密钥
+
+1. 访问 [火山引擎 KMS 控制台](https://console.volcengine.com/kms)
+2. 点击"创建密钥"
+3. 选择"对称密钥",算法选择 `AES_256`
+4. 记录密钥 ID
+
+### 配置 OpenViking
+
+```json
+{
+ "encryption": {
+ "enabled": true,
+ "provider": "volcengine_kms",
+ "volcengine_kms": {
+ "key_id": "d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx",
+ "region": "cn-beijing",
+ "access_key": "AKLTxxxxxxxxxxxxxxxxxx",
+ "secret_key": "Tmpxxxxxxxxxxxxxxxxxxxxxx",
+ "endpoint": null,
+ "key_file": "~/.openviking/openviking-volcengine-root-key.enc"
+ }
+ }
+}
+```
+
+**配置参数说明**:
+
+| 参数 | 说明 | 默认值 |
+|------|------|--------|
+| `key_id` | KMS 密钥 ID | 必需 |
+| `region` | 区域 | 必需 |
+| `access_key` | Access Key | 必需 |
+| `secret_key` | Secret Key | 必需 |
+| `endpoint` | 自定义 KMS 端点(可选) | `null`(使用默认端点) |
+| `key_file` | 加密根密钥本地缓存文件路径 | `"~/.openviking/openviking-volcengine-root-key.enc"` |
+
+### 权限建议
+
+为 Access Key 配置最小权限:
+
+```json
+{
+ "Statement": [
+ {
+ "Effect": "Allow",
+ "Action": [
+ "kms:Encrypt",
+ "kms:Decrypt"
+ ],
+ "Resource": [
+ "trn:kms:*:*:key/d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx"
+ ]
+ }
+ ]
+}
+```
+
+---
+
+## 验证加密
+
+### 方法一:检查文件内容
+
+加密文件以魔术数 `OVE1` 开头:
+
+```bash
+# 查看文件前 4 字节
+hexdump -C ./data/agfs/your-file | head -1
+```
+
+**加密文件**:
+```
+00000000 4f 56 45 31 01 01 00 00 00 20 8a 7b 2c 9d 1e |OVE1..... .{,..|
+```
+(前 4 字节是 `4f 56 45 31` = "OVE1")
+
+**未加密文件**:
+```
+00000000 7b 22 63 6f 6e 74 65 6e 74 73 22 3a 5b 7b 22 70 |{"contents":[{"p|
+```
+
+### 方法二:跨提供程序验证
+
+尝试用不同提供程序解密彼此的数据,应该会失败(这是正常的安全行为):
+
+```python
+# 用 Provider A 加密
+encrypted = await provider_a.encrypt_file_key(plaintext, "test-account")
+
+# 尝试用 Provider B 解密(应该失败)
+try:
+ await provider_b.decrypt_file_key(encrypted, "test-account")
+ print("❌ 安全漏洞:跨提供程序解密成功!")
+except Exception as e:
+ print("✓ 安全:跨提供程序解密失败,符合预期")
+```
+
+---
+
+## 迁移说明
+
+### 从无加密迁移到有加密
+
+1. 备份现有数据
+2. 启用加密(参考上文)
+3. 重新导入所有资源:
+
+```python
+import openviking as ov
+import asyncio
+
+
+async def migrate():
+ client = ov.AsyncOpenViking(path="./data")
+ await client.initialize()
+
+ # 列出所有资源
+ resources = await client.list_resources()
+
+ for resource in resources:
+ # 读取旧资源(未加密)
+ content = await client.read_resource(resource["uri"])
+ # 重新写入(自动加密)
+ await client.add_resource(content, reason="迁移到加密存储")
+
+ await client.close()
+
+
+asyncio.run(migrate())
+```
+
+### 切换密钥提供程序
+
+1. 备份现有数据和密钥
+2. 使用旧提供程序解密所有数据
+3. 配置新提供程序
+4. 重新加密所有数据
+
+**注意**:这是一个破坏性操作,建议在测试环境先验证。
+
+---
+
+## 故障排除
+
+### 密钥文件找不到
+
+```
+Error: Key file not found: ~/.openviking/master.key
+```
+
+**解决方案**:
+1. 检查文件路径是否正确
+2. 使用绝对路径
+3. 确保 `~` 被正确展开(使用 `expanduser()`)
+
+### Vault 连接失败
+
+```
+Error: Failed to connect to Vault
+```
+
+**解决方案**:
+1. 检查 Vault 服务是否运行
+2. 验证 `address` 配置
+3. 检查网络连接和防火墙
+4. 确认 Token 有效且未过期
+
+### 火山 KMS 认证失败
+
+```
+Error: Invalid credentials
+```
+
+**解决方案**:
+1. 检查 Access Key 和 Secret Key 是否正确
+2. 确认密钥有足够权限
+3. 验证区域配置正确
+
+### 跨提供程序解密失败(这是正常的)
+
+```
+Error: KeyMismatchError
+```
+
+**说明**:这是预期的安全行为。不同提供程序使用不同的根密钥,无法相互解密。
+
+### 部分读取返回密文
+
+如果使用旧版本 OpenViking 创建的加密文件,部分读取可能返回密文。
+
+**解决方案**:升级到最新版本的 OpenViking。
+
+---
+
+## 相关文档
+
+- [数据加密](../concepts/10-encryption.md) - 加密概念说明
+- [配置指南](./01-configuration.md) - 完整配置参考
+- [技术设计](../../design/multi-tenant-file-encryption-desigin.md) - 加密技术设计文档
diff --git a/examples/ov.conf.example b/examples/ov.conf.example
index c4211385c..5b837b8db 100644
--- a/examples/ov.conf.example
+++ b/examples/ov.conf.example
@@ -161,12 +161,39 @@
"max_paragraph_length": 1000,
"preserve_line_breaks": false
}
- }
+ },
"encryption": {
"enabled": true,
"provider": "local",
"local": {
"key_file": "~/.openviking/master.key"
}
+ },
+ "encryption_vault_example": {
+ "_comment": "For HashiCorp Vault integration:",
+ "enabled": true,
+ "provider": "vault",
+ "vault": {
+ "address": "http://127.0.0.1:8200",
+ "token": "hvs.your-vault-token",
+ "mount_point": "transit",
+ "kv_mount_point": "secret",
+ "kv_version": 1,
+ "root_key_name": "openviking-root-key",
+ "encrypted_root_key_key": "openviking-encrypted-root-key"
+ }
+ },
+ "encryption_volcengine_kms_example": {
+ "_comment": "For Volcengine KMS integration:",
+ "enabled": true,
+ "provider": "volcengine_kms",
+ "volcengine_kms": {
+ "region": "cn-beijing",
+ "access_key": "AKLTyour-access-key",
+ "secret_key": "your-secret-key",
+ "key_id": "your-kms-key-id",
+ "endpoint": null,
+ "key_file": "~/.openviking/openviking-volcengine-root-key.enc"
+ }
}
}
diff --git a/openviking/crypto/config.py b/openviking/crypto/config.py
index bba2067bc..109e8190f 100644
--- a/openviking/crypto/config.py
+++ b/openviking/crypto/config.py
@@ -67,22 +67,25 @@ def _validate_local_provider_config(config: Dict[str, Any]) -> List[str]:
return errors
# Check if file exists or can be created
- key_file = Path(key_file_path)
+ key_file = Path(key_file_path).expanduser()
if key_file.exists():
# Check permissions
if os.name != "nt": # Skip permission check on Windows
if (key_file.stat().st_mode & 0o077) != 0:
errors.append(f"Key file permissions too open: {key_file_path} (should be 0600)")
else:
- # Check if parent directory is writable
+ # Check if parent directory exists or can be written to
parent_dir = key_file.parent
- if not parent_dir.exists():
- try:
- parent_dir.mkdir(parents=True, exist_ok=True)
- except Exception as e:
- errors.append(f"Cannot create parent directory for key file: {e}")
- elif not os.access(parent_dir, os.W_OK):
- errors.append(f"Cannot create key file at: {key_file_path}")
+ if parent_dir.exists():
+ if not os.access(parent_dir, os.W_OK):
+ errors.append(f"Cannot create key file at: {key_file_path}")
+ else:
+ # Check if we can write to the closest existing ancestor directory
+ current = parent_dir
+ while not current.exists() and current.parent != current:
+ current = current.parent
+ if not os.access(current, os.W_OK):
+ errors.append(f"Cannot create parent directory for key file at: {parent_dir}")
return errors
diff --git a/openviking/crypto/encryptor.py b/openviking/crypto/encryptor.py
index 82943485b..953c2d1d6 100644
--- a/openviking/crypto/encryptor.py
+++ b/openviking/crypto/encryptor.py
@@ -85,12 +85,8 @@ class FileEncryptor:
data_iv = secrets.token_bytes(12)
encrypted_content = await self._aes_gcm_encrypt(file_key, data_iv, plaintext)
- # 3. Encrypt File Key
- key_iv = b""
- if self._provider_type == PROVIDER_LOCAL:
- encrypted_file_key, key_iv = await self.provider.encrypt_file_key(file_key, account_id)
- else:
- encrypted_file_key = await self.provider.encrypt_file_key(file_key, account_id)
+ # 3. Encrypt File Key (all providers now return (encrypted_key, iv)
+ encrypted_file_key, key_iv = await self.provider.encrypt_file_key(file_key, account_id)
# 4. Build Envelope
return self._build_envelope(
@@ -133,13 +129,8 @@ class FileEncryptor:
raise CorruptedCiphertextError(f"Failed to parse envelope: {e}")
try:
- # 3. Decrypt File Key
- if provider_type == PROVIDER_LOCAL:
- file_key = await self.provider.decrypt_file_key(
- encrypted_file_key, key_iv, account_id
- )
- else:
- file_key = await self.provider.decrypt_file_key(encrypted_file_key, account_id)
+ # 3. Decrypt File Key (all providers now use (encrypted_key, iv, account_id)
+ file_key = await self.provider.decrypt_file_key(encrypted_file_key, key_iv, account_id)
except Exception as e:
raise KeyMismatchError(f"Failed to decrypt file key: {e}")
diff --git a/openviking/crypto/providers.py b/openviking/crypto/providers.py
index 9a29f7318..3f8bf68a8 100644
--- a/openviking/crypto/providers.py
+++ b/openviking/crypto/providers.py
@@ -10,6 +10,7 @@ Provides multiple key management methods:
"""
import abc
+import asyncio
import os
import secrets
from abc import ABC
@@ -48,17 +49,103 @@ class RootKeyProvider(ABC):
pass
@abc.abstractmethod
- async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> Any:
+ async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> Tuple[bytes, bytes]:
"""Encrypt File Key."""
pass
@abc.abstractmethod
- async def decrypt_file_key(self, encrypted_key: Any, account_id: str) -> bytes:
+ async def decrypt_file_key(self, encrypted_key: bytes, iv: bytes, account_id: str) -> bytes:
"""Decrypt File Key."""
pass
+ async def _hkdf_derive(
+ self, root_key: bytes, account_id: str, salt: bytes, info_prefix: bytes
+ ) -> bytes:
+ """
+ Derive key using HKDF.
+
+ Args:
+ root_key: Root key
+ account_id: Account ID
+ salt: HKDF salt
+ info_prefix: HKDF info prefix
+
+ Returns:
+ Derived key
+ """
+ try:
+ from cryptography.hazmat.primitives import hashes
+ from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+
+ hkdf = HKDF(
+ algorithm=hashes.SHA256(),
+ length=32,
+ salt=salt,
+ info=info_prefix + account_id.encode(),
+ )
+ return hkdf.derive(root_key)
+ except ImportError:
+ raise ConfigError("cryptography library is required for encryption")
+
+
+class BaseProvider(RootKeyProvider):
+ """Base provider with common encryption functionality."""
+
+ async def _aes_gcm_encrypt(self, key: bytes, iv: bytes, plaintext: bytes) -> bytes:
+ """AES-GCM encryption."""
+ try:
+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+ aesgcm = AESGCM(key)
+ return aesgcm.encrypt(iv, plaintext, associated_data=None)
+ except ImportError:
+ raise ConfigError("cryptography library is required for encryption")
+
+ async def _aes_gcm_decrypt(self, key: bytes, iv: bytes, ciphertext: bytes) -> bytes:
+ """AES-GCM decryption."""
+ try:
+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+ aesgcm = AESGCM(key)
+ return aesgcm.decrypt(iv, ciphertext, associated_data=None)
+ except ImportError:
+ raise ConfigError("cryptography library is required for encryption")
+ except Exception as e:
+ raise AuthenticationFailedError(f"Decryption failed: {e}")
+
+ async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> Tuple[bytes, bytes]:
+ """
+ Encrypt File Key with Account Key.
-class LocalFileProvider(RootKeyProvider):
+ Args:
+ plaintext_key: Plaintext File Key
+ account_id: Account ID
+
+ Returns:
+ (encrypted_key, iv)
+ """
+ account_key = await self.derive_account_key(account_id)
+ iv = secrets.token_bytes(12)
+ encrypted_key = await self._aes_gcm_encrypt(account_key, iv, plaintext_key)
+ return encrypted_key, iv
+
+ async def decrypt_file_key(self, encrypted_key: bytes, iv: bytes, account_id: str) -> bytes:
+ """
+ Decrypt File Key with Account Key.
+
+ Args:
+ encrypted_key: Encrypted File Key
+ iv: Initialization vector
+ account_id: Account ID
+
+ Returns:
+ Decrypted File Key
+ """
+ account_key = await self.derive_account_key(account_id)
+ return await self._aes_gcm_decrypt(account_key, iv, encrypted_key)
+
+
+class LocalFileProvider(BaseProvider):
"""Local file Root Key Provider."""
def __init__(self, key_file: str):
@@ -68,7 +155,7 @@ class LocalFileProvider(RootKeyProvider):
Args:
key_file: Root Key file path
"""
- self.key_file = Path(key_file)
+ self.key_file = Path(key_file).expanduser()
self._root_key: Optional[bytes] = None
async def get_root_key(self) -> bytes:
@@ -103,79 +190,29 @@ class LocalFileProvider(RootKeyProvider):
async def derive_account_key(self, account_id: str) -> bytes:
"""Derive Account Key from Root Key."""
root_key = await self.get_root_key()
- return await self._hkdf_derive(root_key, account_id)
+ return await self._hkdf_derive(root_key, account_id, HKDF_SALT, HKDF_INFO_PREFIX)
- async def _hkdf_derive(self, root_key: bytes, account_id: str) -> bytes:
- """Derive key using HKDF."""
- try:
- from cryptography.hazmat.primitives import hashes
- from cryptography.hazmat.primitives.kdf.hkdf import HKDF
- hkdf = HKDF(
- algorithm=hashes.SHA256(),
- length=32,
- salt=HKDF_SALT,
- info=HKDF_INFO_PREFIX + account_id.encode(),
- )
- return hkdf.derive(root_key)
- except ImportError:
- raise ConfigError("cryptography library is required for encryption")
-
- async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> Tuple[bytes, bytes]:
- """
- Encrypt File Key.
-
- Returns:
- (encrypted_key, iv)
- """
- account_key = await self.derive_account_key(account_id)
- iv = secrets.token_bytes(12)
- encrypted_key = await self._aes_gcm_encrypt(account_key, iv, plaintext_key)
- return encrypted_key, iv
-
- async def decrypt_file_key(self, encrypted_key: bytes, iv: bytes, account_id: str) -> bytes:
- """Decrypt File Key."""
- account_key = await self.derive_account_key(account_id)
- return await self._aes_gcm_decrypt(account_key, iv, encrypted_key)
-
- async def _aes_gcm_encrypt(self, key: bytes, iv: bytes, plaintext: bytes) -> bytes:
- """AES-GCM encryption."""
- try:
- from cryptography.hazmat.primitives.ciphers.aead import AESGCM
-
- aesgcm = AESGCM(key)
- return aesgcm.encrypt(iv, plaintext, associated_data=None)
- except ImportError:
- raise ConfigError("cryptography library is required for encryption")
-
- async def _aes_gcm_decrypt(self, key: bytes, iv: bytes, ciphertext: bytes) -> bytes:
- """AES-GCM decryption."""
- try:
- from cryptography.hazmat.primitives.ciphers.aead import AESGCM
-
- aesgcm = AESGCM(key)
- return aesgcm.decrypt(iv, ciphertext, associated_data=None)
- except ImportError:
- raise ConfigError("cryptography library is required for encryption")
- except Exception as e:
- raise AuthenticationFailedError(f"Decryption failed: {e}")
-
-
-class VaultProvider(RootKeyProvider):
+class VaultProvider(BaseProvider):
"""HashiCorp Vault Key Provider.
Uses HashiCorp Vault's transit secrets engine for key management.
Core features:
- - Root key management: Vault transit secrets engine
+ - Root key management: Vault transit secrets engine (encrypted)
- Account Key derivation: HKDF-SHA256 (from root key)
- - File Key encryption: Vault transit encryption API
+ - File Key encryption: AES-256-GCM (with Account Key)
"""
- SALT = b"OpenViking_KDF_Salt"
- INFO_PREFIX = b"OpenViking_Account_"
- ROOT_KEY_NAME = "openviking-root-key"
-
- def __init__(self, addr: str, token: str, mount_path: str = "transit"):
+ def __init__(
+ self,
+ addr: str,
+ token: str,
+ mount_path: str = "transit",
+ kv_mount_path: str = "secret",
+ kv_version: int = 1,
+ root_key_name: str = "openviking-root-key",
+ encrypted_root_key_key: str = "openviking-encrypted-root-key",
+ ):
"""
Initialize VaultProvider.
@@ -183,10 +220,18 @@ class VaultProvider(RootKeyProvider):
addr: Vault server address (e.g., "http://127.0.0.1:8200")
token: Vault authentication token
mount_path: Transit secrets engine mount path (default: "transit")
+ kv_mount_path: KV secrets engine mount path (default: "secret")
+ kv_version: KV secrets engine version (1 or 2, default: 1)
+ root_key_name: Transit engine key name (default: "openviking-root-key")
+ encrypted_root_key_key: KV engine key path (default: "openviking-encrypted-root-key")
"""
self.addr = addr
self.token = token
self.mount_path = mount_path
+ self.kv_mount_path = kv_mount_path
+ self.kv_version = kv_version
+ self.root_key_name = root_key_name
+ self.encrypted_root_key_key = encrypted_root_key_key
self._client = None
self._root_key: Optional[bytes] = None
@@ -208,7 +253,8 @@ class VaultProvider(RootKeyProvider):
self._client = hvac.Client(url=self.addr, token=self.token)
# Verify Vault is accessible
- if not self._client.is_authenticated():
+ is_auth = await asyncio.to_thread(self._client.is_authenticated)
+ if not is_auth:
raise AuthenticationFailedError("Failed to authenticate with Vault")
# Ensure transit engine is enabled
@@ -223,9 +269,13 @@ class VaultProvider(RootKeyProvider):
"""Ensure transit secrets engine is enabled."""
try:
# Check if transit engine is already enabled
- self._client.sys.list_mounted_secrets_engines()
- if f"{self.mount_path}/" not in self._client.sys.list_mounted_secrets_engines()["data"]:
- self._client.sys.enable_secrets_engine(backend_type="transit", path=self.mount_path)
+ engines = await asyncio.to_thread(self._client.sys.list_mounted_secrets_engines)
+ if f"{self.mount_path}/" not in engines["data"]:
+ await asyncio.to_thread(
+ self._client.sys.enable_secrets_engine,
+ backend_type="transit",
+ path=self.mount_path,
+ )
logger.info(f"Enabled transit secrets engine at {self.mount_path}")
except Exception as e:
logger.warning(f"Failed to check/enable transit engine: {e}")
@@ -234,129 +284,173 @@ class VaultProvider(RootKeyProvider):
"""Ensure root key exists in Vault transit engine."""
try:
# Try to read the key
- self._client.secrets.transit.read_key(
- name=self.ROOT_KEY_NAME, mount_point=self.mount_path
+ await asyncio.to_thread(
+ self._client.secrets.transit.read_key,
+ name=self.root_key_name,
+ mount_point=self.mount_path,
)
except Exception:
# Key doesn't exist, create it
- self._client.secrets.transit.create_key(
- name=self.ROOT_KEY_NAME, key_type="aes256-gcm96", mount_point=self.mount_path
+ await asyncio.to_thread(
+ self._client.secrets.transit.create_key,
+ name=self.root_key_name,
+ key_type="aes256-gcm96",
+ mount_point=self.mount_path,
)
- logger.info(f"Created root key {self.ROOT_KEY_NAME} in Vault")
+ logger.info(f"Created root key {self.root_key_name} in Vault")
- async def get_root_key(self) -> bytes:
+ async def _encrypt_with_vault(self, plaintext: bytes) -> bytes:
"""
- Get root key from Vault.
-
- Note: Since we can't export the key directly from Vault transit,
- we use a fixed seed and derive the root key.
-
- Returns:
- Root key
- """
- if self._root_key is None:
- # Use a fixed seed for root key derivation
- self._root_key = b"OpenViking_Vault_Root_Seed_Key_v1"
- return self._root_key
-
- async def derive_account_key(self, account_id: str) -> bytes:
- """
- Derive Account Key using HKDF.
+ Encrypt data with Vault transit.
Args:
- account_id: Account ID
+ plaintext: Plaintext data
Returns:
- Derived Account Key
+ Encrypted data
"""
- root_key = await self.get_root_key()
- return await self._hkdf_derive(root_key, account_id)
+ client = await self._get_client()
+ import base64
- async def _hkdf_derive(self, root_key: bytes, account_id: str) -> bytes:
+ plaintext_b64 = base64.b64encode(plaintext).decode("utf-8")
+ response = await asyncio.to_thread(
+ client.secrets.transit.encrypt_data,
+ name=self.root_key_name,
+ plaintext=plaintext_b64,
+ mount_point=self.mount_path,
+ )
+ ciphertext_str = response["data"]["ciphertext"]
+ return ciphertext_str.encode("utf-8")
+
+ async def _decrypt_with_vault(self, ciphertext: bytes) -> bytes:
"""
- Derive key using HKDF.
+ Decrypt data with Vault transit.
Args:
- root_key: Root key
- account_id: Account ID
+ ciphertext: Encrypted data
Returns:
- Derived key
+ Decrypted data
"""
- try:
- from cryptography.hazmat.primitives import hashes
- from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+ client = await self._get_client()
+ import base64
- hkdf = HKDF(
- algorithm=hashes.SHA256(),
- length=32,
- salt=self.SALT,
- info=self.INFO_PREFIX + account_id.encode(),
- )
- return hkdf.derive(root_key)
- except ImportError:
- raise ConfigError("cryptography library is required for encryption")
+ ciphertext_str = ciphertext.decode("utf-8")
+ response = await asyncio.to_thread(
+ client.secrets.transit.decrypt_data,
+ name=self.root_key_name,
+ ciphertext=ciphertext_str,
+ mount_point=self.mount_path,
+ )
+ return base64.b64decode(response["data"]["plaintext"])
- async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> bytes:
+ async def _get_or_create_root_key(self) -> bytes:
"""
- Encrypt File Key using Vault transit.
-
- Args:
- plaintext_key: Plaintext File Key
- account_id: Account ID (not used in Vault mode)
+ Get or create root key.
Returns:
- Encrypted File Key as bytes
+ Root key bytes
"""
- client = await self._get_client()
+ if self._root_key is not None:
+ return self._root_key
+ client = await self._get_client()
import base64
- plaintext_b64 = base64.b64encode(plaintext_key).decode("utf-8")
+ try:
+ # Try to read encrypted root key from Vault kv
+ if self.kv_version == 2:
+ response = await asyncio.to_thread(
+ client.secrets.kv.v2.read_secret_version,
+ path=self.encrypted_root_key_key,
+ mount_point=self.kv_mount_path,
+ )
+ encrypted_root_key_b64 = response["data"]["data"]["encrypted_root_key"]
+ else:
+ response = await asyncio.to_thread(
+ client.secrets.kv.v1.read_secret,
+ path=self.encrypted_root_key_key,
+ mount_point=self.kv_mount_path,
+ )
+ encrypted_root_key_b64 = response["data"]["encrypted_root_key"]
- response = client.secrets.transit.encrypt_data(
- name=self.ROOT_KEY_NAME, plaintext=plaintext_b64, mount_point=self.mount_path
- )
+ encrypted_root_key = base64.b64decode(encrypted_root_key_b64)
+ self._root_key = await self._decrypt_with_vault(encrypted_root_key)
+ logger.info("Loaded existing root key from Vault")
+ except Exception:
+ # Generate new root key
+ import secrets
- ciphertext_str = response["data"]["ciphertext"]
- return ciphertext_str.encode("utf-8")
+ self._root_key = secrets.token_bytes(32)
- async def decrypt_file_key(self, encrypted_key: bytes, account_id: str) -> bytes:
- """
- Decrypt File Key using Vault transit.
+ # Encrypt and store root key in Vault kv
+ encrypted_root_key = await self._encrypt_with_vault(self._root_key)
+ try:
+ if self.kv_version == 2:
+ await asyncio.to_thread(
+ client.secrets.kv.v2.create_or_update_secret,
+ path=self.encrypted_root_key_key,
+ mount_point=self.kv_mount_path,
+ secret={
+ "encrypted_root_key": base64.b64encode(encrypted_root_key).decode(
+ "utf-8"
+ )
+ },
+ )
+ else:
+ await asyncio.to_thread(
+ client.secrets.kv.v1.create_or_update_secret,
+ path=self.encrypted_root_key_key,
+ mount_point=self.kv_mount_path,
+ secret={
+ "encrypted_root_key": base64.b64encode(encrypted_root_key).decode(
+ "utf-8"
+ )
+ },
+ )
+ logger.info("Created and stored new root key in Vault")
+ except Exception as e:
+ raise ConfigError(
+ f"Failed to persist root key to Vault. "
+ f"Refusing to start with ephemeral key (data loss risk): {e}"
+ )
- Args:
- encrypted_key: Encrypted File Key (ciphertext bytes from Vault)
- account_id: Account ID (not used in Vault mode)
+ return self._root_key
- Returns:
- Decrypted File Key
+ async def get_root_key(self) -> bytes:
"""
- client = await self._get_client()
+ Get root key.
- ciphertext_str = encrypted_key.decode("utf-8")
+ Returns:
+ Root key
+ """
+ return await self._get_or_create_root_key()
- response = client.secrets.transit.decrypt_data(
- name=self.ROOT_KEY_NAME, ciphertext=ciphertext_str, mount_point=self.mount_path
- )
+ async def derive_account_key(self, account_id: str) -> bytes:
+ """
+ Derive Account Key using HKDF.
- import base64
+ Args:
+ account_id: Account ID
- return base64.b64decode(response["data"]["plaintext"])
+ Returns:
+ Derived Account Key
+ """
+ root_key = await self.get_root_key()
+ return await self._hkdf_derive(root_key, account_id, HKDF_SALT, HKDF_INFO_PREFIX)
-class VolcengineKMSProvider(RootKeyProvider):
+class VolcengineKMSProvider(BaseProvider):
"""Volcengine KMS Key Provider.
Suitable for production environments, using Volcengine KMS service for key management.
Core features:
- - Root key storage: Volcengine KMS
+ - Root key storage: Volcengine KMS (encrypted)
- Account Key derivation: HKDF-SHA256
- - File Key encryption: Volcengine KMS encryption API
+ - File Key encryption: AES-256-GCM (with Account Key)
"""
- SALT = b"OpenViking_KDF_Salt"
- INFO_PREFIX = b"OpenViking_Account_"
+ ROOT_KEY_FILENAME = "openviking-volcengine-root-key.enc"
def __init__(
self,
@@ -365,6 +459,7 @@ class VolcengineKMSProvider(RootKeyProvider):
secret_access_key: str,
key_id: str,
endpoint: Optional[str] = None,
+ key_file: Optional[str] = None,
):
"""
Initialize Volcengine KMS Provider.
@@ -375,13 +470,19 @@ class VolcengineKMSProvider(RootKeyProvider):
secret_access_key: Volcengine Secret Access Key
key_id: Volcengine KMS Key ID (immutable system-generated identifier)
endpoint: Custom KMS service endpoint (optional)
+ key_file: Path to encrypted root key file (default: ~/.openviking/openviking-volcengine-root-key.enc)
"""
self.region = region
self.access_key_id = access_key_id
self.secret_access_key = secret_access_key
self.key_id = key_id
self.endpoint = endpoint or f"kms.{region}.volcengineapi.com"
+ if key_file:
+ self.key_file = Path(key_file).expanduser()
+ else:
+ self.key_file = Path.home() / ".openviking" / self.ROOT_KEY_FILENAME
self._kms_client = None
+ self._root_key: Optional[bytes] = None
async def _get_kms_client(self):
"""
@@ -433,11 +534,13 @@ class VolcengineKMSProvider(RootKeyProvider):
}
return api_info
- def encrypt(self, key_id, plaintext):
+ def encrypt(self, key_id, plaintext, encryption_context=None):
body = {
"KeyID": key_id,
"Plaintext": base64.b64encode(plaintext).decode("utf-8"),
}
+ if encryption_context:
+ body["EncryptionContext"] = encryption_context
res = self.json("Encrypt", {}, json.dumps(body))
if res == "":
@@ -449,11 +552,13 @@ class VolcengineKMSProvider(RootKeyProvider):
return base64.b64decode(res_json["Result"]["CiphertextBlob"])
raise Exception(f"Unexpected response: {res_json}")
- def decrypt(self, ciphertext_blob, key_id):
+ def decrypt(self, ciphertext_blob, key_id, encryption_context=None):
body = {
"KeyID": key_id,
"CiphertextBlob": base64.b64encode(ciphertext_blob).decode("utf-8"),
}
+ if encryption_context:
+ body["EncryptionContext"] = encryption_context
res = self.json("Decrypt", {}, json.dumps(body))
if res == "":
@@ -470,83 +575,96 @@ class VolcengineKMSProvider(RootKeyProvider):
)
return self._kms_client
- async def get_root_key(self) -> bytes:
+ async def _encrypt_with_kms(self, plaintext: bytes) -> bytes:
"""
- Get root key (only used for deriving Account Key).
+ Encrypt data with Volcengine KMS.
- Note: In Volcengine KMS, the root key is managed by KMS service.
- Here we return the base key used for deriving Account Key.
+ Args:
+ plaintext: Plaintext data
Returns:
- Base key
+ Encrypted data
"""
- return b"OpenViking_Volcengine_KMS_Root_Seed"
+ client = await self._get_kms_client()
+ return await asyncio.to_thread(client.encrypt, self.key_id, plaintext)
- async def derive_account_key(self, account_id: str) -> bytes:
+ async def _decrypt_with_kms(self, ciphertext: bytes) -> bytes:
"""
- Derive Account Key using HKDF.
+ Decrypt data with Volcengine KMS.
Args:
- account_id: Account ID
+ ciphertext: Encrypted data
Returns:
- Derived Account Key
+ Decrypted data
"""
- root_key = await self.get_root_key()
- return await self._hkdf_derive(root_key, account_id)
+ client = await self._get_kms_client()
+ return await asyncio.to_thread(client.decrypt, ciphertext, self.key_id)
- async def _hkdf_derive(self, root_key: bytes, account_id: str) -> bytes:
+ async def _get_or_create_root_key(self) -> bytes:
"""
- Derive key using HKDF.
-
- Args:
- root_key: Root key
- account_id: Account ID
+ Get or create root key.
Returns:
- Derived key
+ Root key bytes
"""
+ if self._root_key is not None:
+ return self._root_key
+
try:
- from cryptography.hazmat.primitives import hashes
- from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+ # Try to read encrypted root key from file
+ if self.key_file.exists():
+ with open(self.key_file, "rb") as f:
+ encrypted_root_key = f.read()
+ self._root_key = await self._decrypt_with_kms(encrypted_root_key)
+ logger.info(f"Loaded existing root key from {self.key_file}")
+ return self._root_key
+ except Exception as e:
+ raise ConfigError(
+ f"Failed to load existing root key from {self.key_file}. Cannot continue: {e}"
+ )
- hkdf = HKDF(
- algorithm=hashes.SHA256(),
- length=32,
- salt=self.SALT,
- info=self.INFO_PREFIX + account_id.encode(),
+ # Generate new root key
+ import secrets
+
+ self._root_key = secrets.token_bytes(32)
+
+ # Encrypt and store root key
+ try:
+ encrypted_root_key = await self._encrypt_with_kms(self._root_key)
+ self.key_file.parent.mkdir(parents=True, exist_ok=True)
+ with open(self.key_file, "wb") as f:
+ f.write(encrypted_root_key)
+ logger.info(f"Created and stored new root key at {self.key_file}")
+ except Exception as e:
+ raise ConfigError(
+ f"Failed to persist root key to file. "
+ f"Refusing to start with ephemeral key (data loss risk): {e}"
)
- return hkdf.derive(root_key)
- except ImportError:
- raise ConfigError("cryptography library is required for encryption")
- async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> bytes:
- """
- Encrypt File Key using Volcengine KMS.
+ return self._root_key
- Args:
- plaintext_key: Plaintext File Key
- account_id: Account ID (not used in Volcengine KMS mode)
+ async def get_root_key(self) -> bytes:
+ """
+ Get root key.
Returns:
- Encrypted File Key
+ Root key
"""
- client = await self._get_kms_client()
- return client.encrypt(self.key_id, plaintext_key)
+ return await self._get_or_create_root_key()
- async def decrypt_file_key(self, encrypted_key: bytes, account_id: str) -> bytes:
+ async def derive_account_key(self, account_id: str) -> bytes:
"""
- Decrypt File Key using Volcengine KMS.
+ Derive Account Key using HKDF.
Args:
- encrypted_key: Encrypted File Key
- account_id: Account ID (not used in Volcengine KMS mode)
+ account_id: Account ID
Returns:
- Decrypted File Key
+ Derived Account Key
"""
- client = await self._get_kms_client()
- return client.decrypt(encrypted_key, self.key_id)
+ root_key = await self.get_root_key()
+ return await self._hkdf_derive(root_key, account_id, HKDF_SALT, HKDF_INFO_PREFIX)
def create_root_key_provider(
@@ -576,10 +694,24 @@ def create_root_key_provider(
address = vault_config.get("address")
token = vault_config.get("token")
mount_point = vault_config.get("mount_point", "transit")
+ kv_mount_point = vault_config.get("kv_mount_point", "secret")
+ kv_version = vault_config.get("kv_version", 1)
+ root_key_name = vault_config.get("root_key_name", "openviking-root-key")
+ encrypted_root_key_key = vault_config.get(
+ "encrypted_root_key_key", "openviking-encrypted-root-key"
+ )
if not address or not token:
raise ConfigError("vault.address and vault.token are required")
- return VaultProvider(address, token, mount_point)
+ return VaultProvider(
+ address,
+ token,
+ mount_point,
+ kv_mount_point,
+ kv_version,
+ root_key_name,
+ encrypted_root_key_key,
+ )
elif provider_type == "volcengine_kms":
volc_config = config.get("volcengine_kms", {})
@@ -587,10 +719,14 @@ def create_root_key_provider(
access_key = volc_config.get("access_key")
secret_key = volc_config.get("secret_key")
key_id = volc_config.get("key_id")
+ endpoint = volc_config.get("endpoint")
+ key_file = volc_config.get("key_file")
if not all([region, access_key, secret_key, key_id]):
raise ConfigError("volcengine_kms region, access_key, secret_key, key_id are required")
- return VolcengineKMSProvider(region, access_key, secret_key, key_id)
+ return VolcengineKMSProvider(
+ region, access_key, secret_key, key_id, endpoint=endpoint, key_file=key_file
+ )
else:
raise ConfigError(f"Unsupported provider type: {provider_type}")
diff --git a/openviking/server/api_keys.py b/openviking/server/api_keys.py
index 98b361d4a..636656de8 100644
--- a/openviking/server/api_keys.py
+++ b/openviking/server/api_keys.py
@@ -498,11 +498,10 @@ class APIKeyManager:
raw = content.content if hasattr(content, "content") else b""
# Decrypt content if encryption is enabled
- if self._viking_fs._encryptor:
- # Extract account ID from path
- parts = path.split("/")
- account_id = parts[2] if len(parts) >= 3 else "default"
- raw = await self._viking_fs._encryptor.decrypt(account_id, raw)
+ # Extract account ID from path
+ parts = path.split("/")
+ account_id = parts[2] if len(parts) >= 3 else "default"
+ raw = await self._viking_fs.decrypt_bytes(account_id, raw)
text = raw.decode("utf-8") if isinstance(raw, bytes) else raw
return json.loads(text)
@@ -516,11 +515,10 @@ class APIKeyManager:
content = content.encode("utf-8")
# Encrypt content if encryption is enabled
- if self._viking_fs._encryptor:
- # Extract account ID from path
- parts = path.split("/")
- account_id = parts[2] if len(parts) >= 3 else "default"
- content = await self._viking_fs._encryptor.encrypt(account_id, content)
+ # Extract account ID from path
+ parts = path.split("/")
+ account_id = parts[2] if len(parts) >= 3 else "default"
+ content = await self._viking_fs.encrypt_bytes(account_id, content)
# Ensure parent directories exist
self._ensure_parent_dirs(path)
diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.py
index 100adaf71..744271dce 100644
--- a/openviking/storage/viking_fs.py
+++ b/openviking/storage/viking_fs.py
@@ -204,6 +204,36 @@ class VikingFS:
real_ctx = self._ctx_or_default(ctx)
return await self._encryptor.decrypt(real_ctx.account_id, content)
+ async def encrypt_bytes(self, account_id: str, data: bytes) -> bytes:
+ """
+ Encrypt bytes using the encryptor for the specified account.
+
+ Args:
+ account_id: Account ID to use for encryption
+ data: Bytes to encrypt
+
+ Returns:
+ Encrypted bytes, or original bytes if encryption is disabled
+ """
+ if not self._encryptor:
+ return data
+ return await self._encryptor.encrypt(account_id, data)
+
+ async def decrypt_bytes(self, account_id: str, data: bytes) -> bytes:
+ """
+ Decrypt bytes using the encryptor for the specified account.
+
+ Args:
+ account_id: Account ID to use for decryption
+ data: Bytes to decrypt
+
+ Returns:
+ Decrypted bytes, or original bytes if encryption is disabled
+ """
+ if not self._encryptor:
+ return data
+ return await self._encryptor.decrypt(account_id, data)
+
@contextmanager
def bind_request_context(self, ctx: RequestContext):
"""Temporarily bind ctx for legacy internal call paths without explicit ctx param."""
@@ -258,18 +288,35 @@ class VikingFS:
"""Read file"""
self._ensure_access(uri, ctx)
path = self._uri_to_path(uri, ctx=ctx)
- result = self.agfs.read(path, offset, size)
- if isinstance(result, bytes):
- raw = result
- elif result is not None and hasattr(result, "content"):
- raw = result.content
- else:
- raw = b""
- # If encryption is enabled and not reading from offset, try to decrypt
- if self._encryptor and offset == 0 and size == -1:
+ if self._encryptor:
+ # When encryption is enabled: must read entire file for decryption
+ result = self.agfs.read(path, 0, -1)
+ if isinstance(result, bytes):
+ raw = result
+ elif result is not None and hasattr(result, "content"):
+ raw = result.content
+ else:
+ raw = b""
+
raw = await self._decrypt_content(raw, ctx=ctx)
+ # Apply slicing on decrypted plaintext
+ if offset > 0 or size != -1:
+ if size != -1:
+ raw = raw[offset : offset + size]
+ else:
+ raw = raw[offset:]
+ else:
+ # When not encrypted: normal read
+ result = self.agfs.read(path, offset, size)
+ if isinstance(result, bytes):
+ raw = result
+ elif result is not None and hasattr(result, "content"):
+ raw = result.content
+ else:
+ raw = b""
+
return raw
async def write(
@@ -1558,8 +1605,8 @@ class VikingFS:
else:
raw = b""
- # If encryption is enabled and not reading from offset, try to decrypt
- if self._encryptor and offset == 0 and limit == -1:
+ # If encryption is enabled, always decrypt full file first
+ if self._encryptor:
raw = await self._decrypt_content(raw, ctx=ctx)
text = self._decode_bytes(raw)
@@ -1684,13 +1731,13 @@ class VikingFS:
if len(all_entries) >= node_limit:
break
name = entry.get("name", "")
- # 修改后:通过截断字符串来兼容 7 位或更多位的微秒
+ # After modification: compatible with 7+ digits of microseconds by truncating
raw_time = entry.get("modTime", "")
if raw_time and len(raw_time) > 26 and "+" in raw_time:
- # 处理像 2026-02-21T13:20:23.1470042+08:00 这样的字符串
- # 截断为 2026-02-21T13:20:23.147004+08:00
+ # Handle strings like 2026-02-21T13:20:23.1470042+08:00
+ # Truncate to 2026-02-21T13:20:23.147004+08:00
parts = raw_time.split("+")
- # 保持时间部分最多 26 位 (YYYY-MM-DDTHH:MM:SS.mmmmmm)
+ # Keep time part at most 26 characters (YYYY-MM-DDTHH:MM:SS.mmmmmm)
raw_time = parts[0][:26] + "+" + parts[1]
new_entry = {
"uri": self._path_to_uri(f"{path}/{name}", ctx=ctx),
diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py
index 4ca8d9d9f..6ca667509 100644
--- a/tests/integration/conftest.py
+++ b/tests/integration/conftest.py
@@ -31,6 +31,21 @@ TEST_TMP_DIR = PROJECT_ROOT / "test_data" / "tmp_integration"
GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "")
requires_api_key = pytest.mark.skipif(not GOOGLE_API_KEY, reason="GOOGLE_API_KEY not set")
+# ── Vault integration test helpers ──────────────────────────────────────────
+VAULT_ADDR = os.environ.get("VAULT_ADDR", "http://127.0.0.1:8200")
+VAULT_TOKEN = os.environ.get("VAULT_TOKEN", "")
+requires_vault = pytest.mark.skipif(not VAULT_TOKEN, reason="VAULT_TOKEN not set")
+
+# ── Volcengine KMS integration test helpers ──────────────────────────────────────────
+VOLCENGINE_ACCESS_KEY = os.environ.get("VOLCENGINE_ACCESS_KEY", "")
+VOLCENGINE_SECRET_KEY = os.environ.get("VOLCENGINE_SECRET_KEY", "")
+VOLCENGINE_KMS_KEY_ID = os.environ.get("VOLCENGINE_KMS_KEY_ID", "")
+VOLCENGINE_KMS_REGION = os.environ.get("VOLCENGINE_KMS_REGION", "cn-beijing")
+requires_volcengine_kms = pytest.mark.skipif(
+ not (VOLCENGINE_ACCESS_KEY and VOLCENGINE_SECRET_KEY and VOLCENGINE_KMS_KEY_ID),
+ reason="VOLCENGINE_ACCESS_KEY, VOLCENGINE_SECRET_KEY, or VOLCENGINE_KMS_KEY_ID not set",
+)
+
# (model_name, default_dimension, token_limit)
GEMINI_MODELS = [
("gemini-embedding-2-preview", 3072, 8192),
diff --git a/tests/integration/test_encryption_integration.py b/tests/integration/test_encryption_integration.py
index e5486c0b9..ecbdd5d47 100644
--- a/tests/integration/test_encryption_integration.py
+++ b/tests/integration/test_encryption_integration.py
@@ -1092,6 +1092,110 @@ This is a test skill for verifying encryption functionality.
print("✅ All tests completed!")
print("=" * 80)
+ async def test_read_file_with_offset_and_limit_encryption(
+ self, openviking_service_with_encryption
+ ):
+ """
+ Test read_file() with offset and limit returns correct plaintext when encryption is enabled.
+ Verifies that partial reads (by line) work correctly with encryption.
+ """
+ data = openviking_service_with_encryption
+ svc = data["service"]
+
+ # Create request context
+ from openviking.server.identity import RequestContext, Role
+ from openviking_cli.session.user_id import UserIdentifier
+
+ default_user = UserIdentifier("default", "default", "default")
+ ctx = RequestContext(user=default_user, role=Role.ROOT)
+
+ # Write a multi-line test file
+ test_lines = [
+ "Line 0: This is the first line",
+ "Line 1: Second line content",
+ "Line 2: Third line here",
+ "Line 3: Fourth line",
+ "Line 4: Fifth and final line",
+ ]
+ test_content = "\n".join(test_lines)
+ test_uri = "viking://default/test_multiline.txt"
+
+ await svc.viking_fs.write_file(test_uri, test_content, ctx=ctx)
+
+ # Test 1: Read with offset=0, limit=-1 (full file)
+ full_content = await svc.viking_fs.read_file(test_uri, offset=0, limit=-1, ctx=ctx)
+ assert full_content == test_content, "Full file read should return correct content"
+
+ # Test 2: Read with offset=1, limit=3 (lines 1, 2, 3)
+ partial_content = await svc.viking_fs.read_file(test_uri, offset=1, limit=3, ctx=ctx)
+ expected_lines = test_lines[1:4]
+ expected_content = "\n".join(expected_lines)
+ # read_file() 会在最后一行后添加换行符,所以需要处理这种情况
+ assert partial_content.rstrip("\n") == expected_content, "Partial read failed"
+
+ # Test 3: Read with offset=3, limit=-1 (from line 3 to end)
+ from_line_3 = await svc.viking_fs.read_file(test_uri, offset=3, limit=-1, ctx=ctx)
+ expected_from_line_3 = "\n".join(test_lines[3:])
+ assert from_line_3.rstrip("\n") == expected_from_line_3, "Read from offset failed"
+
+ # Verify file is encrypted on disk
+ agfs_client = svc._agfs_client
+ agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)
+ raw_content = agfs_client.read(agfs_path)
+ assert raw_content.startswith(b"OVE1"), "File should be encrypted"
+
+ print("\n" + "=" * 80)
+ print("✅ read_file() with offset/limit encryption test completed!")
+ print("=" * 80)
+
+ async def test_read_with_offset_and_size_encryption(self, openviking_service_with_encryption):
+ """
+ Test read() with offset and size returns correct plaintext when encryption is enabled.
+ Verifies that partial reads (by byte) work correctly with encryption.
+ """
+ data = openviking_service_with_encryption
+ svc = data["service"]
+
+ # Create request context
+ from openviking.server.identity import RequestContext, Role
+ from openviking_cli.session.user_id import UserIdentifier
+
+ default_user = UserIdentifier("default", "default", "default")
+ ctx = RequestContext(user=default_user, role=Role.ROOT)
+
+ # Write a test file
+ test_content = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+ test_uri = "viking://default/test_bytes.txt"
+
+ await svc.viking_fs.write_file(test_uri, test_content.decode("utf-8"), ctx=ctx)
+
+ # Test 1: Read with offset=0, size=-1 (full file)
+ full_bytes = await svc.viking_fs.read(test_uri, offset=0, size=-1, ctx=ctx)
+ # read() 返回的字节会在末尾多一个换行符,所以使用 rstrip(b"\n")
+ assert full_bytes.rstrip(b"\n") == test_content, (
+ "Full file read should return correct bytes"
+ )
+
+ # Test 2: Read with offset=5, size=10 (bytes 5-14)
+ partial_bytes = await svc.viking_fs.read(test_uri, offset=5, size=10, ctx=ctx)
+ expected_bytes = test_content[5:15]
+ assert partial_bytes == expected_bytes, "Partial read failed"
+
+ # Test 3: Read with offset=10, size=-1 (from byte 10 to end)
+ from_byte_10 = await svc.viking_fs.read(test_uri, offset=10, size=-1, ctx=ctx)
+ expected_from_byte_10 = test_content[10:]
+ assert from_byte_10.rstrip(b"\n") == expected_from_byte_10, "Read from offset failed"
+
+ # Verify file is encrypted on disk
+ agfs_client = svc._agfs_client
+ agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)
+ raw_content = agfs_client.read(agfs_path)
+ assert raw_content.startswith(b"OVE1"), "File should be encrypted"
+
+ print("\n" + "=" * 80)
+ print("✅ read() with offset/size encryption test completed!")
+ print("=" * 80)
+
class TestAddResourceWithSemanticProcessing:
"""
diff --git a/tests/integration/test_vault_encryption_integration.py b/tests/integration/test_vault_encryption_integration.py
new file mode 100644
index 000000000..25925306a
--- /dev/null
+++ b/tests/integration/test_vault_encryption_integration.py
@@ -0,0 +1,716 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Vault encryption integration tests.
+
+Tests encryption functionality with HashiCorp Vault as the key provider.
+Requires VAULT_TOKEN env var to be set.
+Run: pytest tests/integration/test_vault_encryption_integration.py -v -m integration
+"""
+
+import secrets
+from pathlib import Path
+
+import pytest
+import pytest_asyncio
+
+from openviking import AsyncOpenViking
+from openviking.crypto.config import bootstrap_encryption
+from openviking.crypto.encryptor import FileEncryptor
+from openviking.crypto.exceptions import AuthenticationFailedError, ConfigError
+from openviking.crypto.providers import VaultProvider, create_root_key_provider
+from openviking.server.api_keys import APIKeyManager
+from openviking.service.core import OpenVikingService
+from openviking_cli.session.user_id import UserIdentifier
+from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton
+from tests.integration.conftest import VAULT_ADDR, VAULT_TOKEN, requires_vault
+
+pytestmark = [pytest.mark.integration, requires_vault]
+
+# Vault environment configuration
+VAULT_MOUNT_PATH = "transit"
+
+
+class TestVaultProviderIntegration:
+ """Integration tests for VaultProvider."""
+
+ @pytest_asyncio.fixture(scope="function")
+ async def vault_provider(self):
+ """Fixture that provides a VaultProvider instance."""
+ provider = VaultProvider(addr=VAULT_ADDR, token=VAULT_TOKEN, mount_path=VAULT_MOUNT_PATH)
+ yield provider
+
+ @pytest.fixture(scope="function")
+ def vault_config(self):
+ """Fixture that provides Vault configuration."""
+ return {
+ "vault": {
+ "address": VAULT_ADDR,
+ "token": VAULT_TOKEN,
+ "mount_point": VAULT_MOUNT_PATH,
+ }
+ }
+
+ async def test_vault_provider_initialization(self, vault_provider):
+ """Test VaultProvider initialization and Vault connection."""
+ client = await vault_provider._get_client()
+ assert client is not None
+ assert client.is_authenticated()
+
+ async def test_derive_account_key(self, vault_provider):
+ """Test deriving account keys."""
+ account_id = "test-account-vault"
+ account_key = await vault_provider.derive_account_key(account_id)
+
+ assert account_key is not None
+ assert isinstance(account_key, bytes)
+ assert len(account_key) == 32
+
+ async def test_derive_different_account_keys(self, vault_provider):
+ """Test that different accounts get different keys."""
+ account1_key = await vault_provider.derive_account_key("account-1")
+ account2_key = await vault_provider.derive_account_key("account-2")
+
+ assert account1_key != account2_key
+
+ async def test_derive_same_account_key_consistent(self, vault_provider):
+ """Test that same account gets consistent key."""
+ key1 = await vault_provider.derive_account_key("test-account")
+ key2 = await vault_provider.derive_account_key("test-account")
+
+ assert key1 == key2
+
+ async def test_encrypt_decrypt_file_key(self, vault_provider):
+ """Test encrypting and decrypting file keys."""
+ plaintext_key = secrets.token_bytes(32)
+ account_id = "test-account-encrypt"
+
+ encrypted_key, iv = await vault_provider.encrypt_file_key(plaintext_key, account_id)
+
+ assert encrypted_key is not None
+ assert isinstance(encrypted_key, bytes)
+ assert encrypted_key != plaintext_key
+ assert iv is not None
+ assert isinstance(iv, bytes)
+
+ decrypted = await vault_provider.decrypt_file_key(encrypted_key, iv, account_id)
+
+ assert decrypted == plaintext_key
+
+ async def test_encrypt_decrypt_multiple_keys(self, vault_provider):
+ """Test encrypting and decrypting multiple different keys."""
+ key1 = secrets.token_bytes(32)
+ key2 = secrets.token_bytes(32)
+ account_id = "test-account-multiple"
+
+ encrypted1, iv1 = await vault_provider.encrypt_file_key(key1, account_id)
+ encrypted2, iv2 = await vault_provider.encrypt_file_key(key2, account_id)
+
+ assert encrypted1 != encrypted2
+
+ decrypted1 = await vault_provider.decrypt_file_key(encrypted1, iv1, account_id)
+ decrypted2 = await vault_provider.decrypt_file_key(encrypted2, iv2, account_id)
+
+ assert decrypted1 == key1
+ assert decrypted2 == key2
+
+ async def test_create_root_key_provider_vault(self, vault_config):
+ """Test creating VaultProvider via create_root_key_provider."""
+ provider = create_root_key_provider("vault", vault_config)
+
+ assert provider is not None
+ assert isinstance(provider, VaultProvider)
+
+ async def test_get_root_key(self, vault_provider):
+ """Test getting root key."""
+ root_key = await vault_provider.get_root_key()
+
+ assert root_key is not None
+ assert isinstance(root_key, bytes)
+ assert len(root_key) > 0
+
+ async def test_full_roundtrip(self, vault_provider):
+ """Test full encryption/decryption roundtrip."""
+ # Derive account key
+ account_id = "test-account-roundtrip"
+ account_key = await vault_provider.derive_account_key(account_id)
+
+ assert account_key is not None
+ assert len(account_key) == 32
+
+ # Generate and encrypt file key
+ file_key = secrets.token_bytes(32)
+ encrypted_key, iv = await vault_provider.encrypt_file_key(file_key, account_id)
+
+ assert encrypted_key is not None
+ assert isinstance(encrypted_key, bytes)
+ assert iv is not None
+ assert isinstance(iv, bytes)
+
+ # Decrypt and verify
+ decrypted_key = await vault_provider.decrypt_file_key(encrypted_key, iv, account_id)
+
+ assert decrypted_key == file_key
+
+
+class TestVaultProviderErrors:
+ """Test error handling for VaultProvider."""
+
+ async def test_invalid_token(self):
+ """Test with invalid token."""
+ provider = VaultProvider(
+ addr="http://127.0.0.1:8200", token="invalid-token", mount_path="transit"
+ )
+
+ with pytest.raises(AuthenticationFailedError):
+ await provider._get_client()
+
+ async def test_invalid_address(self):
+ """Test with invalid address."""
+ provider = VaultProvider(
+ addr="http://invalid-address-that-does-not-exist:8200",
+ token="test-token",
+ mount_path="transit",
+ )
+
+ with pytest.raises(Exception): # noqa: B017
+ await provider._get_client()
+
+ def test_create_provider_missing_config(self):
+ """Test creating provider with missing config."""
+ with pytest.raises(ConfigError):
+ create_root_key_provider("vault", {})
+
+ def test_create_provider_missing_token(self):
+ """Test creating provider with missing token."""
+ with pytest.raises(ConfigError):
+ create_root_key_provider("vault", {"vault": {"address": "http://127.0.0.1:8200"}})
+
+ def test_create_provider_missing_address(self):
+ """Test creating provider with missing address."""
+ with pytest.raises(ConfigError):
+ create_root_key_provider("vault", {"vault": {"token": "test-token"}})
+
+
+@pytest_asyncio.fixture(scope="function")
+async def vault_encryption_config():
+ """Fixture that provides encryption configuration with Vault provider"""
+ return {
+ "encryption": {
+ "enabled": True,
+ "provider": "vault",
+ "vault": {"address": VAULT_ADDR, "token": VAULT_TOKEN, "mount_point": VAULT_MOUNT_PATH},
+ }
+ }
+
+
+@pytest_asyncio.fixture(scope="function")
+async def vault_file_encryptor():
+ """Fixture that provides a FileEncryptor instance with Vault provider for testing"""
+ provider = VaultProvider(addr=VAULT_ADDR, token=VAULT_TOKEN, mount_path=VAULT_MOUNT_PATH)
+ return FileEncryptor(provider)
+
+
+@pytest_asyncio.fixture(scope="function")
+async def openviking_client_with_vault_encryption(test_data_dir: Path, vault_encryption_config):
+ """Fixture that provides an OpenViking client with Vault encryption enabled"""
+ await AsyncOpenViking.reset()
+ OpenVikingConfigSingleton.reset_instance()
+
+ # Clean data directory
+ if test_data_dir.exists():
+ import shutil
+
+ shutil.rmtree(test_data_dir)
+ test_data_dir.mkdir(parents=True, exist_ok=True)
+
+ # Create config dict with encryption enabled
+ config_dict = {}
+ config_dict.update(vault_encryption_config)
+ config_dict["storage"] = {
+ "workspace": str(test_data_dir / "workspace"),
+ "vectordb": {"name": "test", "backend": "local", "project": "default"},
+ }
+ config_dict["embedding"] = {
+ "dense": {
+ "model": "doubao-embedding-vision-250615",
+ "api_key": "2a2284d9-79a3-4978-b718-dfeffdda04ad",
+ "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+ "dimension": 1024,
+ "provider": "volcengine",
+ }
+ }
+
+ # Initialize config singleton
+ OpenVikingConfigSingleton.initialize(config_dict=config_dict)
+
+ client = AsyncOpenViking(path=str(test_data_dir))
+ await client.initialize()
+
+ yield client
+
+ await client.close()
+ await AsyncOpenViking.reset()
+ OpenVikingConfigSingleton.reset_instance()
+
+
+class TestVaultEncryptionBootstrap:
+ """Tests for encryption module bootstrap with Vault provider"""
+
+ async def test_bootstrap_encryption_vault_enabled(self, vault_encryption_config):
+ """Test bootstrapping encryption with Vault provider enabled"""
+ encryptor = await bootstrap_encryption(vault_encryption_config)
+ assert encryptor is not None
+ assert isinstance(encryptor, FileEncryptor)
+ assert isinstance(encryptor.provider, VaultProvider)
+
+
+class TestVaultFileEncryptorIntegration:
+ """Tests for FileEncryptor integration with Vault provider"""
+
+ async def test_encrypt_decrypt_roundtrip_vault(self, vault_file_encryptor):
+ """Test basic encryption/decryption roundtrip with Vault"""
+ account_id = "test_account_vault"
+ plaintext = b"This is a test file content for Vault encryption integration"
+
+ # Encrypt
+ ciphertext = await vault_file_encryptor.encrypt(account_id, plaintext)
+ assert ciphertext != plaintext
+ assert ciphertext.startswith(b"OVE1")
+
+ # Decrypt
+ decrypted = await vault_file_encryptor.decrypt(account_id, ciphertext)
+ assert decrypted == plaintext
+
+ async def test_different_accounts_isolation(self, vault_file_encryptor):
+ """Test that different accounts cannot decrypt each other's data"""
+ account1 = "account_1_vault"
+ account2 = "account_2_vault"
+ plaintext = b"Sensitive data for account isolation test"
+
+ # Encrypt with account1
+ ciphertext = await vault_file_encryptor.encrypt(account1, plaintext)
+
+ # Decrypt with account1 should work
+ decrypted1 = await vault_file_encryptor.decrypt(account1, ciphertext)
+ assert decrypted1 == plaintext
+
+ # Decrypt with account2 should fail
+ from openviking.crypto.exceptions import KeyMismatchError
+
+ with pytest.raises(KeyMismatchError):
+ await vault_file_encryptor.decrypt(account2, ciphertext)
+
+ async def test_encrypt_empty_data_vault(self, vault_file_encryptor):
+ """Test encrypting and decrypting empty data with Vault"""
+ account_id = "test_account_vault"
+ plaintext = b""
+
+ ciphertext = await vault_file_encryptor.encrypt(account_id, plaintext)
+ assert ciphertext.startswith(b"OVE1")
+
+ decrypted = await vault_file_encryptor.decrypt(account_id, ciphertext)
+ assert decrypted == b""
+
+
+class TestVikingFSEncryptionWithVault:
+ """
+ Complete VikingFS encryption tests using Vault as key provider.
+ Includes account management, resource operations, skill operations, etc.
+ """
+
+ ROOT_KEY = "test-root-key-for-vault-encryption-tests-abcdef123456"
+
+ @pytest_asyncio.fixture(scope="function")
+ async def openviking_service_with_vault_encryption(
+ self, test_data_dir: Path, vault_encryption_config
+ ):
+ """
+ Fixture provides OpenVikingService instance with Vault encryption.
+ Also initializes APIKeyManager for account management.
+ """
+ # Clean data directory
+ if test_data_dir.exists():
+ import shutil
+
+ shutil.rmtree(test_data_dir)
+ test_data_dir.mkdir(parents=True, exist_ok=True)
+
+ # Create config dict with encryption enabled and root_api_key set
+ config_dict = {}
+ config_dict.update(vault_encryption_config)
+ config_dict["storage"] = {
+ "workspace": str(test_data_dir / "workspace"),
+ "vectordb": {"name": "test", "backend": "local", "project": "default"},
+ }
+ config_dict["embedding"] = {
+ "dense": {
+ "model": "doubao-embedding-vision-250615",
+ "api_key": "2a2284d9-79a3-4978-b718-dfeffdda04ad",
+ "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+ "dimension": 1024,
+ "provider": "volcengine",
+ }
+ }
+ config_dict["server"] = {"root_api_key": self.ROOT_KEY}
+
+ # Initialize config singleton
+ OpenVikingConfigSingleton.initialize(config_dict=config_dict)
+
+ # Create OpenVikingService
+ svc = OpenVikingService(
+ path=str(test_data_dir / "viking"), user=UserIdentifier.the_default_user("test_user")
+ )
+ await svc.initialize()
+
+ # Create APIKeyManager using VikingFS to ensure system files are encrypted
+ api_key_manager = APIKeyManager(
+ root_key=self.ROOT_KEY, viking_fs=svc.viking_fs, encryption_enabled=True
+ )
+ await api_key_manager.load()
+
+ yield {"service": svc, "api_key_manager": api_key_manager, "test_data_dir": test_data_dir}
+
+ await svc.close()
+ await AsyncOpenViking.reset()
+ OpenVikingConfigSingleton.reset_instance()
+
+ def _is_file_encrypted(self, file_path: Path) -> bool:
+ """
+ Check if file is encrypted (by "OVE1" header).
+
+ Args:
+ file_path: File path
+
+ Returns:
+ True if file is encrypted, False otherwise
+ """
+ if not file_path.exists():
+ return False
+ try:
+ with open(file_path, "rb") as f:
+ header = f.read(4)
+ return header == b"OVE1"
+ except Exception:
+ return False
+
+ async def _check_all_files_encrypted(
+ self, svc, ctx, base_uri: str, print_paths: bool = True
+ ) -> None:
+ """
+ Recursively check if all files in directory are encrypted.
+
+ Args:
+ svc: OpenVikingService instance
+ ctx: RequestContext instance
+ base_uri: Base URI to check
+ print_paths: Whether to print encrypted file paths
+ """
+ agfs_client = svc._agfs_client
+
+ async def _check_recursive(uri: str):
+ try:
+ entries = await svc.viking_fs.ls(uri, ctx=ctx)
+ for entry in entries:
+ entry_uri = entry["uri"]
+ if entry["isDir"]:
+ await _check_recursive(entry_uri)
+ else:
+ # Use VikingFS's _uri_to_path method to get file path
+ agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)
+ raw_content = agfs_client.read(agfs_path)
+ assert raw_content.startswith(b"OVE1"), f"File {entry_uri} is not encrypted"
+ if print_paths:
+ print(f"✓ File is encrypted: {entry_uri}")
+ except Exception as e:
+ if "not found" in str(e).lower():
+ pass
+ else:
+ raise
+
+ await _check_recursive(base_uri)
+
+ @pytest.mark.asyncio
+ async def test_resource_operations_with_encryption(
+ self, openviking_service_with_vault_encryption, tmp_path
+ ):
+ """Test 4: Resource operations and encryption verification"""
+ data = openviking_service_with_vault_encryption
+ svc = data["service"]
+ api_key_manager = data["api_key_manager"]
+
+ # Create test account
+ account_id = "test-account-resources"
+ admin_user_id = "admin"
+ await api_key_manager.create_account(account_id, admin_user_id)
+
+ from openviking.server.identity import RequestContext, Role
+
+ user = UserIdentifier(account_id, admin_user_id, admin_user_id)
+ ctx = RequestContext(user=user, role=Role.ADMIN)
+
+ # Create test resource file
+ test_content = "This is test resource content for Vault encryption"
+ test_uri = f"viking://{account_id}/resources/test_file.txt"
+
+ # Write file
+ await svc.viking_fs.write_file(test_uri, test_content, ctx=ctx)
+
+ # Verify read content is correct
+ read_content = await svc.viking_fs.read_file(test_uri, ctx=ctx)
+ assert read_content == test_content
+
+ # Verify file is encrypted
+ agfs_client = svc._agfs_client
+ agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)
+ raw_content = agfs_client.read(agfs_path)
+ assert raw_content.startswith(b"OVE1")
+
+ # Test various operations
+ resources_dir_uri = f"viking://{account_id}/resources"
+
+ # ls operation
+ ls_entries = await svc.viking_fs.ls(resources_dir_uri, ctx=ctx)
+ assert len(ls_entries) > 0
+
+ # tree operation
+ tree_entries = await svc.viking_fs.tree(resources_dir_uri, ctx=ctx)
+ assert len(tree_entries) > 0
+
+ # grep operation
+ try:
+ grep_result = await svc.viking_fs.grep(resources_dir_uri, "Vault", ctx=ctx)
+ assert grep_result is not None
+ except Exception as e:
+ print(f"[WARNING] grep operation may not be supported: {e}")
+
+ # abstract operation
+ try:
+ abstract = await svc.viking_fs.abstract(test_uri, ctx=ctx)
+ assert abstract is not None
+ assert "OVE1" not in abstract
+ except Exception as e:
+ print(f"[WARNING] abstract operation may not be supported: {e}")
+
+ # overview operation
+ try:
+ overview = await svc.viking_fs.overview(test_uri, ctx=ctx)
+ assert overview is not None
+ assert "OVE1" not in overview
+ except Exception as e:
+ print(f"[WARNING] overview operation may not be supported: {e}")
+
+ @pytest.mark.asyncio
+ async def test_skill_operations_with_encryption(self, openviking_service_with_vault_encryption):
+ """Test 5: Skill operations and encryption verification"""
+ data = openviking_service_with_vault_encryption
+ svc = data["service"]
+ api_key_manager = data["api_key_manager"]
+
+ # Create test account
+ account_id = "test-account-skills"
+ admin_user_id = "admin"
+ await api_key_manager.create_account(account_id, admin_user_id)
+
+ from openviking.server.identity import RequestContext, Role
+
+ user = UserIdentifier(account_id, admin_user_id, admin_user_id)
+ ctx = RequestContext(user=user, role=Role.ADMIN)
+
+ # Create skill directory and file
+ skill_dir_uri = f"viking://{account_id}/agent/test-skill"
+ skill_md_uri = f"{skill_dir_uri}/SKILL.md"
+
+ # Create directory
+ await svc.viking_fs.mkdir(skill_dir_uri, ctx=ctx)
+
+ # Write skill file
+ skill_content = """---
+name: Test Skill
+version: 1.0.0
+description: Test skill for Vault encryption
+---
+
+# Test Skill
+
+This is a test skill for verifying Vault encryption.
+"""
+ await svc.viking_fs.write_file(skill_md_uri, skill_content, ctx=ctx)
+
+ # Verify read content is correct
+ read_content = await svc.viking_fs.read_file(skill_md_uri, ctx=ctx)
+ assert "Test Skill" in read_content
+
+ # Verify file is encrypted
+ agfs_client = svc._agfs_client
+ agfs_path = svc.viking_fs._uri_to_path(skill_md_uri, ctx=ctx)
+ raw_content = agfs_client.read(agfs_path)
+ assert raw_content.startswith(b"OVE1")
+
+ # Test various operations
+ agent_dir_uri = f"viking://{account_id}/agent"
+
+ # ls operation
+ ls_entries = await svc.viking_fs.ls(agent_dir_uri, ctx=ctx)
+ assert len(ls_entries) > 0
+
+ # tree operation
+ try:
+ tree_entries = await svc.viking_fs.tree(agent_dir_uri, ctx=ctx)
+ assert len(tree_entries) > 0
+ except Exception as e:
+ print(f"[WARNING] tree operation may not be supported: {e}")
+
+ @pytest.mark.asyncio
+ async def test_memory_operations_with_encryption(
+ self, openviking_service_with_vault_encryption
+ ):
+ """Test 6: Memory operations and encryption verification"""
+ data = openviking_service_with_vault_encryption
+ svc = data["service"]
+ api_key_manager = data["api_key_manager"]
+
+ # Create test account
+ account_id = "test-account-memories"
+ user_id = "test-user"
+ await api_key_manager.create_account(account_id, user_id)
+
+ from openviking.server.identity import RequestContext, Role
+
+ user = UserIdentifier(account_id, user_id, user_id)
+ ctx = RequestContext(user=user, role=Role.USER)
+
+ # Create memory directory and file
+ memory_dir_uri = f"viking://{account_id}/user/{user_id}/memories"
+ memory_file_uri = f"{memory_dir_uri}/preferences.md"
+
+ # Create directory
+ try:
+ await svc.viking_fs.mkdir(memory_dir_uri, ctx=ctx)
+ except Exception:
+ pass # Directory may already exist
+
+ # Write memory file
+ memory_content = "# User Preferences\n\nTheme: dark\nLanguage: English"
+ await svc.viking_fs.write_file(memory_file_uri, memory_content, ctx=ctx)
+
+ # Verify read content is correct
+ read_content = await svc.viking_fs.read_file(memory_file_uri, ctx=ctx)
+ assert "User Preferences" in read_content
+
+ # Verify file is encrypted
+ agfs_client = svc._agfs_client
+ agfs_path = svc.viking_fs._uri_to_path(memory_file_uri, ctx=ctx)
+ raw_content = agfs_client.read(agfs_path)
+ assert raw_content.startswith(b"OVE1")
+
+ @pytest.mark.asyncio
+ async def test_session_operations_with_encryption(
+ self, openviking_service_with_vault_encryption
+ ):
+ """Test 7: Session operations and encryption verification"""
+ data = openviking_service_with_vault_encryption
+ svc = data["service"]
+ api_key_manager = data["api_key_manager"]
+
+ # Create test account
+ account_id = "test-account-sessions"
+ user_id = "test-user"
+ await api_key_manager.create_account(account_id, user_id)
+
+ from openviking.server.identity import RequestContext, Role
+
+ user = UserIdentifier(account_id, user_id, user_id)
+ ctx = RequestContext(user=user, role=Role.USER)
+
+ # Create session
+ session = await svc.sessions.create(ctx)
+ session_id = session.session_id
+ assert session_id is not None
+
+ # Check if session directory files are encrypted
+ session_dir_uri = f"viking://{account_id}/session"
+ await self._check_all_files_encrypted(svc, ctx, session_dir_uri, print_paths=False)
+
+ @pytest.mark.asyncio
+ async def test_complete_encryption_workflow_with_vault(
+ self, openviking_service_with_vault_encryption
+ ):
+ """
+ Complete encryption workflow test using Vault as key provider, including:
+ 1. Precondition: Create random account and user
+ 2. Execution: Resource, skill, session, relationship operations
+ 3. Encryption verification: Check OVE1 header through file system
+ 4. Postcondition: Delete account
+ """
+ import uuid
+
+ from openviking.server.identity import RequestContext, Role
+
+ svc = openviking_service_with_vault_encryption["service"]
+ api_key_manager = openviking_service_with_vault_encryption["api_key_manager"]
+ openviking_service_with_vault_encryption["test_data_dir"]
+
+ # 1. Precondition: Create random account and user
+ random_account_id = f"test-account-{uuid.uuid4()}"
+ random_user_id = f"test-user-{uuid.uuid4()}"
+ print(f"\n=== Creating account: {random_account_id}, user: {random_user_id} ===")
+
+ await api_key_manager.create_account(random_account_id, random_user_id)
+ user = UserIdentifier(random_account_id, random_user_id, random_user_id)
+ ctx = RequestContext(user=user, role=Role.ADMIN)
+
+ # 2. Execution: Resource, skill, session operations
+ print("\n=== Executing operations ===")
+
+ # Create resource
+ test_content = f"Test content for account {random_account_id} with Vault encryption"
+ test_resource_uri = f"viking://{random_account_id}/resources/test_workflow.txt"
+ await svc.viking_fs.write_file(test_resource_uri, test_content, ctx=ctx)
+ print(f"✓ Resource created successfully: {test_resource_uri}")
+
+ # Create skill
+ skill_dir_uri = f"viking://{random_account_id}/agent/test-workflow-skill"
+ skill_md_uri = f"{skill_dir_uri}/SKILL.md"
+ await svc.viking_fs.mkdir(skill_dir_uri, ctx=ctx)
+ skill_content = """---
+name: Workflow Test Skill
+version: 1.0.0
+description: Test skill for complete workflow
+---
+
+# Workflow Test Skill
+"""
+ await svc.viking_fs.write_file(skill_md_uri, skill_content, ctx=ctx)
+ print(f"✓ Skill created successfully: {skill_md_uri}")
+
+ # Create memory
+ memory_dir_uri = f"viking://{random_account_id}/user/{random_user_id}/memories"
+ memory_file_uri = f"{memory_dir_uri}/test_memory.md"
+ try:
+ await svc.viking_fs.mkdir(memory_dir_uri, ctx=ctx)
+ except Exception:
+ pass
+ memory_content = "# Test Memory\n\nThis is a test memory for workflow"
+ await svc.viking_fs.write_file(memory_file_uri, memory_content, ctx=ctx)
+ print(f"✓ Memory created successfully: {memory_file_uri}")
+
+ # Create session
+ session = await svc.sessions.create(ctx)
+ session_id = session.session_id
+ print(f"✓ Session created successfully: {session_id}")
+
+ # 3. Encryption verification: Check OVE1 header through file system
+ print("\n=== Encryption verification ===")
+
+ # Check if all files are encrypted
+ account_root_uri = f"viking://{random_account_id}"
+ await self._check_all_files_encrypted(svc, ctx, account_root_uri, print_paths=True)
+
+ # 4. Postcondition: Delete account
+ print("\n=== Cleanup ===")
+ await api_key_manager.delete_account(random_account_id)
+ print(f"✓ Account {random_account_id} deleted")
+
+ print("\n✓ Complete Vault encryption workflow test completed!")
diff --git a/tests/integration/test_volcengine_kms_encryption_integration.py b/tests/integration/test_volcengine_kms_encryption_integration.py
new file mode 100644
index 000000000..ea85e6fab
--- /dev/null
+++ b/tests/integration/test_volcengine_kms_encryption_integration.py
@@ -0,0 +1,779 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+Volcengine KMS Encryption Integration Tests
+
+Tests encryption functionality integrated with VikingFS and OpenViking service
+using Volcengine KMS as the key provider.
+Requires VOLCENGINE_ACCESS_KEY, VOLCENGINE_SECRET_KEY, and VOLCENGINE_KMS_KEY_ID env vars.
+Run: pytest tests/integration/test_volcengine_kms_encryption_integration.py -v -m integration
+"""
+
+import secrets
+import tempfile
+from pathlib import Path
+
+import pytest
+import pytest_asyncio
+
+from openviking import AsyncOpenViking
+from openviking.crypto.encryptor import FileEncryptor
+from openviking.crypto.providers import VolcengineKMSProvider
+from openviking.server.api_keys import APIKeyManager
+from openviking.service.core import OpenVikingService
+from openviking_cli.session.user_id import UserIdentifier
+from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton
+from tests.integration.conftest import (
+ VOLCENGINE_ACCESS_KEY,
+ VOLCENGINE_KMS_KEY_ID,
+ VOLCENGINE_KMS_REGION,
+ VOLCENGINE_SECRET_KEY,
+ requires_volcengine_kms,
+)
+
+pytestmark = [pytest.mark.integration, requires_volcengine_kms]
+
+
+class TestVolcengineKMSProvider:
+ """Tests for VolcengineKMSProvider"""
+
+ @pytest_asyncio.fixture(scope="module")
+ async def volcengine_kms_provider(self):
+ """
+ Fixture that provides a VolcengineKMSProvider instance for testing.
+ Requires valid Volcengine KMS credentials from environment variables.
+ """
+ provider = VolcengineKMSProvider(
+ region=VOLCENGINE_KMS_REGION,
+ access_key_id=VOLCENGINE_ACCESS_KEY,
+ secret_access_key=VOLCENGINE_SECRET_KEY,
+ key_id=VOLCENGINE_KMS_KEY_ID,
+ )
+ return provider
+
+ @pytest.mark.asyncio
+ async def test_initialize_provider(self, volcengine_kms_provider):
+ """Test that VolcengineKMSProvider initializes correctly"""
+ assert volcengine_kms_provider is not None
+ assert volcengine_kms_provider.region == "cn-beijing"
+ assert volcengine_kms_provider.key_id == "d926aa0d-deee-455a-9186-0d07e2e250e1"
+
+ @pytest.mark.asyncio
+ async def test_get_root_key(self, volcengine_kms_provider):
+ """Test getting root key"""
+ root_key = await volcengine_kms_provider.get_root_key()
+ assert isinstance(root_key, bytes)
+ assert len(root_key) > 0
+
+ @pytest.mark.asyncio
+ async def test_derive_account_key(self, volcengine_kms_provider):
+ """Test deriving account key"""
+ account_id = "test-account-volcengine"
+ account_key = await volcengine_kms_provider.derive_account_key(account_id)
+ assert isinstance(account_key, bytes)
+ assert len(account_key) == 32 # Should be 256-bit key
+
+ @pytest.mark.asyncio
+ async def test_derive_different_account_keys(self, volcengine_kms_provider):
+ """Test that different account IDs produce different keys"""
+ account1 = "account-1-volc"
+ account2 = "account-2-volc"
+
+ key1 = await volcengine_kms_provider.derive_account_key(account1)
+ key2 = await volcengine_kms_provider.derive_account_key(account2)
+
+ assert key1 != key2
+
+ @pytest.mark.asyncio
+ async def test_derive_same_account_key(self, volcengine_kms_provider):
+ """Test that same account ID produces same key"""
+ account_id = "test-account-consistent"
+
+ key1 = await volcengine_kms_provider.derive_account_key(account_id)
+ key2 = await volcengine_kms_provider.derive_account_key(account_id)
+
+ assert key1 == key2
+
+ @pytest.mark.asyncio
+ async def test_encrypt_decrypt_file_key(self, volcengine_kms_provider):
+ """Test encrypting and decrypting a file key"""
+ plaintext_key = secrets.token_bytes(32) # 256-bit key
+ account_id = "test-account-encrypt"
+
+ encrypted_key, iv = await volcengine_kms_provider.encrypt_file_key(
+ plaintext_key, account_id
+ )
+ assert isinstance(encrypted_key, bytes)
+ assert isinstance(iv, bytes)
+
+ decrypted = await volcengine_kms_provider.decrypt_file_key(encrypted_key, iv, account_id)
+ assert decrypted == plaintext_key
+
+ @pytest.mark.asyncio
+ async def test_encrypt_different_keys(self, volcengine_kms_provider):
+ """Test that different plaintexts produce different ciphertexts"""
+ key1 = secrets.token_bytes(32)
+ key2 = secrets.token_bytes(32)
+ account_id = "test-account-multi"
+
+ encrypted1, iv1 = await volcengine_kms_provider.encrypt_file_key(key1, account_id)
+ encrypted2, iv2 = await volcengine_kms_provider.encrypt_file_key(key2, account_id)
+
+ assert encrypted1 != encrypted2
+
+ @pytest.mark.asyncio
+ async def test_full_roundtrip(self, volcengine_kms_provider):
+ """Test complete roundtrip: derive account key, encrypt, decrypt"""
+ account_id = "test-account-roundtrip"
+
+ # Derive account key
+ account_key = await volcengine_kms_provider.derive_account_key(account_id)
+ assert isinstance(account_key, bytes)
+ assert len(account_key) == 32
+
+ # Generate and encrypt file key
+ file_key = secrets.token_bytes(32)
+ encrypted_file_key, iv = await volcengine_kms_provider.encrypt_file_key(
+ file_key, account_id
+ )
+
+ # Decrypt file key
+ decrypted_file_key = await volcengine_kms_provider.decrypt_file_key(
+ encrypted_file_key, iv, account_id
+ )
+ assert decrypted_file_key == file_key
+
+
+@pytest_asyncio.fixture(scope="function")
+async def volcengine_kms_config():
+ """Fixture that provides Volcengine KMS encryption configuration"""
+ return {
+ "encryption": {
+ "enabled": True,
+ "provider": "volcengine_kms",
+ "volcengine_kms": {
+ "region": VOLCENGINE_KMS_REGION,
+ "access_key": VOLCENGINE_ACCESS_KEY,
+ "secret_key": VOLCENGINE_SECRET_KEY,
+ "key_id": VOLCENGINE_KMS_KEY_ID,
+ },
+ }
+ }
+
+
+@pytest_asyncio.fixture(scope="function")
+async def volcengine_file_encryptor():
+ """Fixture that provides a FileEncryptor with Volcengine KMS"""
+ provider = VolcengineKMSProvider(
+ region=VOLCENGINE_KMS_REGION,
+ access_key_id=VOLCENGINE_ACCESS_KEY,
+ secret_access_key=VOLCENGINE_SECRET_KEY,
+ key_id=VOLCENGINE_KMS_KEY_ID,
+ )
+ return FileEncryptor(provider)
+
+
+@pytest_asyncio.fixture(scope="function")
+async def openviking_client_with_volcengine_encryption(test_data_dir: Path, volcengine_kms_config):
+ """Fixture that provides an OpenViking client with Volcengine KMS encryption"""
+ await AsyncOpenViking.reset()
+ OpenVikingConfigSingleton.reset_instance()
+
+ if test_data_dir.exists():
+ import shutil
+
+ shutil.rmtree(test_data_dir)
+ test_data_dir.mkdir(parents=True, exist_ok=True)
+
+ config_dict = {}
+ config_dict.update(volcengine_kms_config)
+ config_dict["storage"] = {
+ "workspace": str(test_data_dir / "workspace"),
+ "vectordb": {"name": "test", "backend": "local", "project": "default"},
+ }
+ config_dict["embedding"] = {
+ "dense": {
+ "model": "doubao-embedding-vision-250615",
+ "api_key": "2a2284d9-79a3-4978-b718-dfeffdda04ad",
+ "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+ "dimension": 1024,
+ "provider": "volcengine",
+ }
+ }
+
+ OpenVikingConfigSingleton.initialize(config_dict=config_dict)
+
+ client = AsyncOpenViking(path=str(test_data_dir))
+ await client.initialize()
+
+ yield client
+
+ await client.close()
+ await AsyncOpenViking.reset()
+ OpenVikingConfigSingleton.reset_instance()
+
+
+class TestVolcengineKMSEncryptionBootstrap:
+ """Tests for encryption module bootstrap with Volcengine KMS"""
+
+ @pytest.mark.asyncio
+ async def test_bootstrap_encryption_with_volcengine_kms(self, volcengine_kms_config, tmp_path):
+ """Test bootstrapping encryption with Volcengine KMS"""
+ from openviking.crypto.config import bootstrap_encryption
+
+ config_dict = volcengine_kms_config
+ config_dict["storage"] = {"workspace": str(tmp_path)}
+
+ result = await bootstrap_encryption(config_dict)
+
+ assert result is not None
+ assert isinstance(result, FileEncryptor)
+
+ @pytest.mark.asyncio
+ async def test_file_encryptor_with_volcengine_kms(self, volcengine_file_encryptor):
+ """Test FileEncryptor with Volcengine KMS"""
+ account_id = "test_account_volcengine"
+ plaintext = b"Test data for Volcengine KMS encryption"
+
+ ciphertext = await volcengine_file_encryptor.encrypt(account_id, plaintext)
+ assert ciphertext.startswith(b"OVE1")
+
+ decrypted = await volcengine_file_encryptor.decrypt(account_id, ciphertext)
+ assert decrypted == plaintext
+
+ @pytest.mark.asyncio
+ async def test_different_accounts_isolation(self, volcengine_file_encryptor):
+ """Test that different accounts cannot decrypt each other's data"""
+ account1 = "account_1_volc"
+ account2 = "account_2_volc"
+ plaintext = b"Sensitive data for account isolation test"
+
+ # Encrypt with account1
+ ciphertext = await volcengine_file_encryptor.encrypt(account1, plaintext)
+
+ # Decrypt with account1 should work
+ decrypted1 = await volcengine_file_encryptor.decrypt(account1, ciphertext)
+ assert decrypted1 == plaintext
+
+ # Decrypt with account2 should fail
+ from openviking.crypto.exceptions import KeyMismatchError
+
+ with pytest.raises(KeyMismatchError):
+ await volcengine_file_encryptor.decrypt(account2, ciphertext)
+
+ @pytest.mark.asyncio
+ async def test_encrypt_empty_data(self, volcengine_file_encryptor):
+ """Test encrypting and decrypting empty data"""
+ account_id = "test_account_volc"
+ plaintext = b""
+
+ ciphertext = await volcengine_file_encryptor.encrypt(account_id, plaintext)
+ assert ciphertext.startswith(b"OVE1")
+
+ decrypted = await volcengine_file_encryptor.decrypt(account_id, ciphertext)
+ assert decrypted == b""
+
+
+class TestVolcengineKMSEncryptionDisabled:
+ """Tests for behavior when encryption is disabled"""
+
+ @pytest_asyncio.fixture(scope="function")
+ async def openviking_client_without_encryption(self, test_data_dir: Path):
+ """Fixture that provides an OpenViking client without encryption"""
+ await AsyncOpenViking.reset()
+ OpenVikingConfigSingleton.reset_instance()
+
+ if test_data_dir.exists():
+ import shutil
+
+ shutil.rmtree(test_data_dir)
+ test_data_dir.mkdir(parents=True, exist_ok=True)
+
+ # Create config dict with encryption disabled
+ config_dict = {
+ "encryption": {"enabled": False},
+ "storage": {
+ "workspace": str(test_data_dir / "workspace"),
+ "vectordb": {"name": "test", "backend": "local", "project": "default"},
+ },
+ "embedding": {
+ "dense": {
+ "model": "doubao-embedding-vision-250615",
+ "api_key": "2a2284d9-79a3-4978-b718-dfeffdda04ad",
+ "api_base": "https://ark.cn-beijing.volces.com/api/v3",
+ "dimension": 1024,
+ "provider": "volcengine",
+ }
+ },
+ }
+
+ # Initialize config singleton
+ OpenVikingConfigSingleton.initialize(config_dict=config_dict)
+
+ client = AsyncOpenViking(path=str(test_data_dir))
+ await client.initialize()
+
+ yield client
+
+ await client.close()
+ await AsyncOpenViking.reset()
+ OpenVikingConfigSingleton.reset_instance()
+
+ @pytest.mark.asyncio
+ async def test_read_write_without_encryption(
+ self, openviking_client_without_encryption: AsyncOpenViking, tmp_path: Path
+ ):
+ """Test normal file operations when encryption is disabled"""
+ client = openviking_client_without_encryption
+
+ test_file = tmp_path / "normal_file.txt"
+ test_content = "Normal content without encryption"
+ test_file.write_text(test_content)
+
+ result = await client.add_resource(
+ path=str(test_file), reason="Normal operation test", wait=True
+ )
+ root_uri = result["root_uri"]
+
+ # Get tree structure to find the actual file
+ uris = await client.tree(root_uri)
+ assert len(uris) > 0
+
+ # Find the actual file (skip .abstract.md and .overview.md)
+ found = False
+ for data in uris:
+ if not data["isDir"]:
+ filename = data["name"]
+ # Skip auto-generated files
+ if filename not in [".abstract.md", ".overview.md"]:
+ file_uri = data["uri"]
+ content = await client.read(file_uri)
+ assert content == test_content
+ found = True
+ break
+ assert found, "Could not find the test file"
+
+
+class TestVikingFSEncryptionWithVolcengineKMS:
+ """
+ Complete VikingFS encryption tests using Volcengine KMS as key provider
+ """
+
+ ROOT_KEY = "test-root-key-for-volcengine-encryption-tests"
+
+ @pytest_asyncio.fixture(scope="function")
+ async def openviking_service_with_volcengine_encryption(self, test_data_dir: Path):
+ """Fixture that provides OpenVikingService with Volcengine KMS encryption"""
+ await AsyncOpenViking.reset()
+ OpenVikingConfigSingleton.reset_instance()
+
+ if test_data_dir.exists():
+ import shutil
+
+ shutil.rmtree(test_data_dir)
+ test_data_dir.mkdir(parents=True, exist_ok=True)
+
+ svc = OpenVikingService(
+ path=str(test_data_dir / "viking"), user=UserIdentifier.the_default_user("test_user")
+ )
+ await svc.initialize()
+
+ api_key_manager = APIKeyManager(
+ root_key=self.ROOT_KEY, viking_fs=svc.viking_fs, encryption_enabled=True
+ )
+ await api_key_manager.load()
+
+ yield {"service": svc, "api_key_manager": api_key_manager, "test_data_dir": test_data_dir}
+
+ await svc.close()
+ await AsyncOpenViking.reset()
+ OpenVikingConfigSingleton.reset_instance()
+
+ def _is_file_encrypted(self, file_path: Path) -> bool:
+ """Check if file is encrypted (by "OVE1" header)"""
+ if not file_path.exists():
+ return False
+ try:
+ with open(file_path, "rb") as f:
+ header = f.read(4)
+ return header == b"OVE1"
+ except Exception:
+ return False
+
+ async def _check_all_files_encrypted(
+ self, svc, ctx, base_uri: str, print_paths: bool = True
+ ) -> None:
+ """Recursively check if all files in directory are encrypted"""
+ agfs_client = svc._agfs_client
+
+ async def _check_recursive(uri: str):
+ try:
+ entries = await svc.viking_fs.ls(uri, ctx=ctx)
+ for entry in entries:
+ entry_uri = entry["uri"]
+ if entry["isDir"]:
+ await _check_recursive(entry_uri)
+ else:
+ # Use VikingFS's _uri_to_path method to get file path
+ agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)
+ raw_content = agfs_client.read(agfs_path)
+ assert raw_content.startswith(b"OVE1"), f"File {entry_uri} is not encrypted"
+ if print_paths:
+ print(f"✓ File is encrypted: {entry_uri}")
+ except Exception as e:
+ if "not found" in str(e).lower():
+ pass
+ else:
+ raise
+
+ await _check_recursive(base_uri)
+
+ @pytest.mark.asyncio
+ async def test_bootstrap_encryption_with_volcengine_kms(
+ self, openviking_service_with_volcengine_encryption
+ ):
+ """Test 1: Bootstrapping encryption with Volcengine KMS"""
+ data = openviking_service_with_volcengine_encryption
+ api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
+
+ account_id = "test-account-volcengine-bootstrap"
+ admin_user_id = "admin"
+ user_key = await api_key_manager.create_account(account_id, admin_user_id)
+
+ assert user_key is not None
+ assert len(user_key) == 64
+
+ agfs_data_root = test_data_dir / "viking" / "viking"
+
+ global_accounts_path = agfs_data_root / "_system" / "accounts.json"
+ assert global_accounts_path.exists()
+ assert self._is_file_encrypted(global_accounts_path)
+
+ @pytest.mark.asyncio
+ async def test_file_encryption_with_volcengine_kms(
+ self, openviking_service_with_volcengine_encryption
+ ):
+ """Test 2: Encrypting files with Volcengine KMS"""
+ data = openviking_service_with_volcengine_encryption
+ svc = data["service"]
+ api_key_manager = data["api_key_manager"]
+ test_data_dir = data["test_data_dir"]
+
+ account_id = "test-account-volcengine-file"
+ admin_user_id = "admin"
+ await api_key_manager.create_account(account_id, admin_user_id)
+
+ from openviking.server.identity import RequestContext, Role
+
+ user = UserIdentifier(account_id, admin_user_id, admin_user_id)
+ ctx = RequestContext(user=user, role=Role.ADMIN)
+
+ test_content = "Test content with Volcengine KMS encryption"
+ with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:
+ f.write(test_content)
+ temp_file_path = f.name
+
+ try:
+ resource = await svc.resources.add_resource(
+ path=temp_file_path, reason="Test Volcengine KMS encryption", ctx=ctx, wait=True
+ )
+ resource["root_uri"]
+
+ agfs_data_root = test_data_dir / "viking" / "viking"
+ resources_dir = agfs_data_root / account_id / "resources"
+
+ assert resources_dir.exists()
+
+ await self._check_all_files_encrypted(svc, ctx, f"viking:///{account_id}/resources")
+
+ finally:
+ import os
+
+ os.unlink(temp_file_path)
+
+ @pytest.mark.asyncio
+ async def test_resource_operations_with_encryption(
+ self, openviking_service_with_volcengine_encryption, tmp_path
+ ):
+ """Test 4: Resource operations and encryption verification"""
+ data = openviking_service_with_volcengine_encryption
+ svc = data["service"]
+ api_key_manager = data["api_key_manager"]
+
+ # Create test account
+ account_id = "test-account-resources"
+ admin_user_id = "admin"
+ await api_key_manager.create_account(account_id, admin_user_id)
+
+ from openviking.server.identity import RequestContext, Role
+
+ user = UserIdentifier(account_id, admin_user_id, admin_user_id)
+ ctx = RequestContext(user=user, role=Role.ADMIN)
+
+ # Create test resource file
+ test_content = "This is test resource content for Volcengine KMS encryption"
+ test_uri = f"viking://{account_id}/resources/test_file.txt"
+
+ # Write file
+ await svc.viking_fs.write_file(test_uri, test_content, ctx=ctx)
+
+ # Verify read content is correct
+ read_content = await svc.viking_fs.read_file(test_uri, ctx=ctx)
+ assert read_content == test_content
+
+ # Verify file is encrypted
+ agfs_client = svc._agfs_client
+ agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)
+ raw_content = agfs_client.read(agfs_path)
+ assert raw_content.startswith(b"OVE1")
+
+ # Test various operations
+ resources_dir_uri = f"viking://{account_id}/resources"
+
+ # ls operation
+ ls_entries = await svc.viking_fs.ls(resources_dir_uri, ctx=ctx)
+ assert len(ls_entries) > 0
+
+ # tree operation
+ tree_entries = await svc.viking_fs.tree(resources_dir_uri, ctx=ctx)
+ assert len(tree_entries) > 0
+
+ # grep operation
+ try:
+ grep_result = await svc.viking_fs.grep(resources_dir_uri, "Volcengine", ctx=ctx)
+ assert grep_result is not None
+ except Exception as e:
+ print(f"[WARNING] grep operation may not be supported: {e}")
+
+ # abstract operation
+ try:
+ abstract = await svc.viking_fs.abstract(test_uri, ctx=ctx)
+ assert abstract is not None
+ assert "OVE1" not in abstract
+ except Exception as e:
+ print(f"[WARNING] abstract operation may not be supported: {e}")
+
+ # overview operation
+ try:
+ overview = await svc.viking_fs.overview(test_uri, ctx=ctx)
+ assert overview is not None
+ assert "OVE1" not in overview
+ except Exception as e:
+ print(f"[WARNING] overview operation may not be supported: {e}")
+
+ @pytest.mark.asyncio
+ async def test_skill_operations_with_encryption(
+ self, openviking_service_with_volcengine_encryption
+ ):
+ """Test 5: Skill operations and encryption verification"""
+ data = openviking_service_with_volcengine_encryption
+ svc = data["service"]
+ api_key_manager = data["api_key_manager"]
+
+ # Create test account
+ account_id = "test-account-skills"
+ admin_user_id = "admin"
+ await api_key_manager.create_account(account_id, admin_user_id)
+
+ from openviking.server.identity import RequestContext, Role
+
+ user = UserIdentifier(account_id, admin_user_id, admin_user_id)
+ ctx = RequestContext(user=user, role=Role.ADMIN)
+
+ # Create skill directory and file
+ skill_dir_uri = f"viking://{account_id}/agent/test-skill"
+ skill_md_uri = f"{skill_dir_uri}/SKILL.md"
+
+ # Create directory
+ await svc.viking_fs.mkdir(skill_dir_uri, ctx=ctx)
+
+ # Write skill file
+ skill_content = """---
+name: Test Skill
+version: 1.0.0
+description: Test skill for Volcengine KMS encryption
+---
+
+# Test Skill
+
+This is a test skill for verifying Volcengine KMS encryption.
+"""
+ await svc.viking_fs.write_file(skill_md_uri, skill_content, ctx=ctx)
+
+ # Verify read content is correct
+ read_content = await svc.viking_fs.read_file(skill_md_uri, ctx=ctx)
+ assert "Test Skill" in read_content
+
+ # Verify file is encrypted
+ agfs_client = svc._agfs_client
+ agfs_path = svc.viking_fs._uri_to_path(skill_md_uri, ctx=ctx)
+ raw_content = agfs_client.read(agfs_path)
+ assert raw_content.startswith(b"OVE1")
+
+ # Test various operations
+ agent_dir_uri = f"viking://{account_id}/agent"
+
+ # ls operation
+ ls_entries = await svc.viking_fs.ls(agent_dir_uri, ctx=ctx)
+ assert len(ls_entries) > 0
+
+ # tree operation
+ try:
+ tree_entries = await svc.viking_fs.tree(agent_dir_uri, ctx=ctx)
+ assert len(tree_entries) > 0
+ except Exception as e:
+ print(f"[WARNING] tree operation may not be supported: {e}")
+
+ @pytest.mark.asyncio
+ async def test_memory_operations_with_encryption(
+ self, openviking_service_with_volcengine_encryption
+ ):
+ """Test 6: Memory operations and encryption verification"""
+ data = openviking_service_with_volcengine_encryption
+ svc = data["service"]
+ api_key_manager = data["api_key_manager"]
+
+ # Create test account
+ account_id = "test-account-memories"
+ user_id = "test-user"
+ await api_key_manager.create_account(account_id, user_id)
+
+ from openviking.server.identity import RequestContext, Role
+
+ user = UserIdentifier(account_id, user_id, user_id)
+ ctx = RequestContext(user=user, role=Role.USER)
+
+ # Create memory directory and file
+ memory_dir_uri = f"viking://{account_id}/user/{user_id}/memories"
+ memory_file_uri = f"{memory_dir_uri}/preferences.md"
+
+ # Create directory
+ try:
+ await svc.viking_fs.mkdir(memory_dir_uri, ctx=ctx)
+ except Exception:
+ pass # Directory may already exist
+
+ # Write memory file
+ memory_content = "# User Preferences\n\nTheme: dark\nLanguage: English"
+ await svc.viking_fs.write_file(memory_file_uri, memory_content, ctx=ctx)
+
+ # Verify read content is correct
+ read_content = await svc.viking_fs.read_file(memory_file_uri, ctx=ctx)
+ assert "User Preferences" in read_content
+
+ # Verify file is encrypted
+ agfs_client = svc._agfs_client
+ agfs_path = svc.viking_fs._uri_to_path(memory_file_uri, ctx=ctx)
+ raw_content = agfs_client.read(agfs_path)
+ assert raw_content.startswith(b"OVE1")
+
+ @pytest.mark.asyncio
+ async def test_session_operations_with_encryption(
+ self, openviking_service_with_volcengine_encryption
+ ):
+ """Test 7: Session operations and encryption verification"""
+ data = openviking_service_with_volcengine_encryption
+ svc = data["service"]
+ api_key_manager = data["api_key_manager"]
+
+ # Create test account
+ account_id = "test-account-sessions"
+ user_id = "test-user"
+ await api_key_manager.create_account(account_id, user_id)
+
+ from openviking.server.identity import RequestContext, Role
+
+ user = UserIdentifier(account_id, user_id, user_id)
+ ctx = RequestContext(user=user, role=Role.USER)
+
+ # Create session
+ session = await svc.sessions.create(ctx)
+ session_id = session.session_id
+ assert session_id is not None
+
+ # Check if session directory files are encrypted
+ session_dir_uri = f"viking://{account_id}/session"
+ await self._check_all_files_encrypted(svc, ctx, session_dir_uri, print_paths=False)
+
+ @pytest.mark.asyncio
+ async def test_complete_encryption_workflow(
+ self, openviking_service_with_volcengine_encryption, tmp_path
+ ):
+ """Test 8: Complete encryption workflow using Volcengine KMS as key provider"""
+ data = openviking_service_with_volcengine_encryption
+ svc = data["service"]
+ api_key_manager = data["api_key_manager"]
+
+ # Generate random account name
+ import secrets
+
+ random_suffix = secrets.token_hex(4)
+ test_account_id = f"test-account-workflow-{random_suffix}"
+ test_user_id = "admin"
+
+ # Create account
+ await api_key_manager.create_account(test_account_id, test_user_id)
+
+ from openviking.server.identity import RequestContext, Role
+
+ user = UserIdentifier(test_account_id, test_user_id, test_user_id)
+ ctx = RequestContext(user=user, role=Role.ADMIN)
+
+ print(f"\n✓ Created test account: {test_account_id}")
+
+ # 1. Resource operations
+ print("\n1. Testing resource operations...")
+ resource_uri = f"viking://{test_account_id}/resources/test_workflow.txt"
+ resource_content = "Test resource content for complete workflow"
+ await svc.viking_fs.write_file(resource_uri, resource_content, ctx=ctx)
+ read_content = await svc.viking_fs.read_file(resource_uri, ctx=ctx)
+ assert read_content == resource_content
+ print("✓ Resource operations test passed")
+
+ # 2. Skill operations
+ print("\n2. Testing skill operations...")
+ skill_dir_uri = f"viking://{test_account_id}/agent/test-workflow-skill"
+ skill_md_uri = f"{skill_dir_uri}/SKILL.md"
+ await svc.viking_fs.mkdir(skill_dir_uri, ctx=ctx)
+ skill_content = """---
+name: Workflow Test Skill
+version: 1.0.0
+description: Test skill for complete workflow
+---
+
+# Workflow Test Skill
+"""
+ await svc.viking_fs.write_file(skill_md_uri, skill_content, ctx=ctx)
+ read_skill = await svc.viking_fs.read_file(skill_md_uri, ctx=ctx)
+ assert "Workflow Test Skill" in read_skill
+ print("✓ Skill operations test passed")
+
+ # 3. Memory operations
+ print("\n3. Testing memory operations...")
+ memory_dir_uri = f"viking://{test_account_id}/user/{test_user_id}/memories"
+ memory_file_uri = f"{memory_dir_uri}/test_memory.md"
+ try:
+ await svc.viking_fs.mkdir(memory_dir_uri, ctx=ctx)
+ except Exception:
+ pass
+ memory_content = "# Test Memory\n\nThis is a test memory for workflow"
+ await svc.viking_fs.write_file(memory_file_uri, memory_content, ctx=ctx)
+ read_memory = await svc.viking_fs.read_file(memory_file_uri, ctx=ctx)
+ assert "Test Memory" in read_memory
+ print("✓ Memory operations test passed")
+
+ # 4. Session operations
+ print("\n4. Testing session operations...")
+ session = await svc.sessions.create(ctx)
+ assert session.session_id is not None
+ print("✓ Session operations test passed")
+
+ # 5. Verify all files are encrypted
+ print("\n5. Verifying all files are encrypted...")
+ account_root_uri = f"viking://{test_account_id}"
+ await self._check_all_files_encrypted(svc, ctx, account_root_uri, print_paths=False)
+ print("✓ All files are encrypted")
+
+ print("\n✓ Complete Volcengine KMS encryption workflow test passed!")
diff --git a/tests/unit/crypto/test_providers.py b/tests/unit/crypto/test_local_provider.py
similarity index 100%
rename from tests/unit/crypto/test_providers.py
rename to tests/unit/crypto/test_local_provider.py
diff --git a/tests/unit/crypto/test_providers_mock.py b/tests/unit/crypto/test_providers_mock.py
new file mode 100644
index 000000000..514b4df7e
--- /dev/null
+++ b/tests/unit/crypto/test_providers_mock.py
@@ -0,0 +1,275 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: Apache-2.0
+"""
+Mock-based unit tests for all KeyProvider implementations.
+
+These tests do NOT require external services (Vault, Volcengine KMS) to run.
+They use mocking to verify:
+1. Envelope format compatibility
+2. Provider-type byte round-tripping
+3. API interaction correctness
+4. Error handling
+"""
+
+import base64
+import os
+import secrets
+import struct
+import tempfile
+from unittest.mock import AsyncMock, Mock, patch
+
+import pytest
+
+from openviking.crypto.encryptor import FileEncryptor
+from openviking.crypto.providers import (
+ PROVIDER_LOCAL,
+ PROVIDER_VAULT,
+ PROVIDER_VOLCENGINE,
+ LocalFileProvider,
+ VaultProvider,
+ VolcengineKMSProvider,
+)
+
+
+def local_file_provider():
+ """Create a LocalFileProvider instance with temporary file."""
+ with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
+ root_key = secrets.token_bytes(32)
+ f.write(root_key.hex())
+ temp_path = f.name
+ os.chmod(temp_path, 0o600)
+
+ try:
+ provider = LocalFileProvider(key_file=temp_path)
+ yield provider
+ finally:
+ if os.path.exists(temp_path):
+ os.unlink(temp_path)
+
+
+def vault_mock_provider():
+ """Create a VaultProvider with mocked methods."""
+ provider = VaultProvider(
+ addr="http://mock-vault:8200", token="mock-token", mount_path="transit"
+ )
+
+ # Mock all the methods that would call external services
+ provider._get_client = AsyncMock()
+ provider._ensure_transit_engine_enabled = AsyncMock()
+ provider._ensure_root_key_exists = AsyncMock()
+
+ yield provider
+
+
+def volcengine_mock_provider():
+ """Create a VolcengineKMSProvider with mocked methods."""
+ provider = VolcengineKMSProvider(
+ access_key_id="mock-access-key",
+ secret_access_key="mock-secret-key",
+ region="cn-beijing",
+ key_id="mock-key-id",
+ )
+
+ # Mock all the methods that would call external services
+ provider._get_kms_client = AsyncMock()
+
+ yield provider
+
+
+class TestEnvelopeFormat:
+ """Tests for envelope format and provider-type byte round-tripping."""
+
+ @pytest.mark.asyncio
+ async def test_local_provider_envelope_format(self, local_file_provider):
+ """Test envelope format with LocalFileProvider."""
+ encryptor = FileEncryptor(local_file_provider)
+ account_id = "test-account"
+ plaintext = b"This is a test message"
+
+ # Encrypt
+ ciphertext = await encryptor.encrypt(account_id, plaintext)
+
+ # Verify envelope starts with magic
+ assert ciphertext.startswith(b"OVE1")
+
+ # Verify provider type is in envelope
+ assert ciphertext[5] == PROVIDER_LOCAL
+
+ # Decrypt back
+ decrypted = await encryptor.decrypt(account_id, ciphertext)
+ assert decrypted == plaintext
+
+ @pytest.mark.asyncio
+ async def test_vault_provider_type_detection(self):
+ """Test that VaultProvider type is correctly detected by FileEncryptor."""
+ # Create a minimal mock provider that just needs type detection
+ mock_provider = Mock(spec=VaultProvider)
+
+ # FileEncryptor should detect it as Vault type
+ encryptor = FileEncryptor(mock_provider)
+ assert encryptor._provider_type == PROVIDER_VAULT
+
+ @pytest.mark.asyncio
+ async def test_volcengine_provider_type_detection(self):
+ """Test that VolcengineKMSProvider type is correctly detected by FileEncryptor."""
+ # Create a minimal mock provider that just needs type detection
+ mock_provider = Mock(spec=VolcengineKMSProvider)
+
+ # FileEncryptor should detect it as Volcengine type
+ encryptor = FileEncryptor(mock_provider)
+ assert encryptor._provider_type == PROVIDER_VOLCENGINE
+
+ @pytest.mark.asyncio
+ async def test_provider_type_roundtrip(self, local_file_provider):
+ """Test that provider type is correctly preserved in envelope."""
+ encryptor = FileEncryptor(local_file_provider)
+ account_id = "test-account"
+ plaintext = b"Roundtrip test"
+
+ ciphertext = await encryptor.encrypt(account_id, plaintext)
+
+ # Parse the envelope manually to verify provider type
+ HEADER_SIZE = 12
+ assert len(ciphertext) >= HEADER_SIZE
+
+ (magic, version, provider_type, _, _, _) = struct.unpack(
+ "!4sBBHHH", ciphertext[:HEADER_SIZE]
+ )
+
+ assert magic == b"OVE1"
+ assert version == 0x01
+ assert provider_type == PROVIDER_LOCAL
+
+
+class TestVaultProviderMock:
+ """Tests for VaultProvider with mocked methods."""
+
+ @pytest.mark.asyncio
+ async def test_vault_encrypt_decrypt_with_mock(self, vault_mock_provider):
+ """Test encrypt_file_key and decrypt_file_key can be mocked."""
+ account_id = "test-account"
+ plaintext_key = secrets.token_bytes(32)
+ mock_ciphertext = b"vault:v1:encrypted_data"
+ mock_iv = secrets.token_bytes(12)
+
+ # Setup mock methods
+ vault_mock_provider.encrypt_file_key = AsyncMock(return_value=(mock_ciphertext, mock_iv))
+ vault_mock_provider.decrypt_file_key = AsyncMock(return_value=plaintext_key)
+
+ # Encrypt
+ encrypted, iv = await vault_mock_provider.encrypt_file_key(plaintext_key, account_id)
+ assert encrypted == mock_ciphertext
+ assert iv == mock_iv
+ vault_mock_provider.encrypt_file_key.assert_called_once_with(plaintext_key, account_id)
+
+ # Decrypt
+ decrypted = await vault_mock_provider.decrypt_file_key(encrypted, iv, account_id)
+ assert decrypted == plaintext_key
+ vault_mock_provider.decrypt_file_key.assert_called_once_with(encrypted, iv, account_id)
+
+ @pytest.mark.asyncio
+ async def test_vault_provider_with_file_encryptor(self, vault_mock_provider):
+ """Test FileEncryptor works with mocked VaultProvider."""
+ # Setup mock methods
+ test_file_key = secrets.token_bytes(32)
+ test_iv = secrets.token_bytes(12)
+ vault_mock_provider.get_root_key = AsyncMock(return_value=b"test_root_key")
+ vault_mock_provider.derive_account_key = AsyncMock(return_value=test_file_key)
+ vault_mock_provider.encrypt_file_key = AsyncMock(return_value=(b"encrypted_fk", test_iv))
+ vault_mock_provider.decrypt_file_key = AsyncMock(return_value=test_file_key)
+
+ encryptor = FileEncryptor(vault_mock_provider)
+ account_id = "test-account"
+ plaintext = b"Test with Vault mock"
+
+ # Patch the AES encryption to simplify
+ with (
+ patch.object(
+ encryptor, "_aes_gcm_encrypt", new=AsyncMock(return_value=b"encrypted_data")
+ ),
+ patch.object(encryptor, "_aes_gcm_decrypt", new=AsyncMock(return_value=plaintext)),
+ ):
+ ciphertext = await encryptor.encrypt(account_id, plaintext)
+ assert ciphertext.startswith(b"OVE1")
+ assert ciphertext[5] == PROVIDER_VAULT
+
+
+class TestVolcengineKMSProviderMock:
+ """Tests for VolcengineKMSProvider with mocked methods."""
+
+ @pytest.mark.asyncio
+ async def test_volcengine_encrypt_decrypt_with_mock(self, volcengine_mock_provider):
+ """Test encrypt_file_key and decrypt_file_key can be mocked."""
+ account_id = "test-account"
+ plaintext_key = secrets.token_bytes(32)
+ mock_ciphertext = base64.b64encode(b"encrypted_data")
+ mock_iv = secrets.token_bytes(12)
+
+ # Setup mock methods
+ volcengine_mock_provider.encrypt_file_key = AsyncMock(
+ return_value=(mock_ciphertext, mock_iv)
+ )
+ volcengine_mock_provider.decrypt_file_key = AsyncMock(return_value=plaintext_key)
+
+ # Encrypt
+ encrypted, iv = await volcengine_mock_provider.encrypt_file_key(plaintext_key, account_id)
+ assert encrypted == mock_ciphertext
+ assert iv == mock_iv
+ volcengine_mock_provider.encrypt_file_key.assert_called_once_with(plaintext_key, account_id)
+
+ # Decrypt
+ decrypted = await volcengine_mock_provider.decrypt_file_key(encrypted, iv, account_id)
+ assert decrypted == plaintext_key
+ volcengine_mock_provider.decrypt_file_key.assert_called_once_with(encrypted, iv, account_id)
+
+
+class TestCrossProviderEnvelope:
+ """Tests for cross-provider envelope behavior."""
+
+ @pytest.mark.asyncio
+ async def test_different_provider_type_in_envelope(self):
+ """Test that different providers set different type bytes in envelope."""
+ # Create mock providers
+ mock_local = Mock(spec=LocalFileProvider)
+ mock_vault = Mock(spec=VaultProvider)
+ mock_volcengine = Mock(spec=VolcengineKMSProvider)
+
+ # Each should be detected as different types
+ assert FileEncryptor(mock_local)._provider_type == PROVIDER_LOCAL
+ assert FileEncryptor(mock_vault)._provider_type == PROVIDER_VAULT
+ assert FileEncryptor(mock_volcengine)._provider_type == PROVIDER_VOLCENGINE
+
+ @pytest.mark.asyncio
+ async def test_same_provider_different_instance_can_decrypt(self, local_file_provider):
+ """Test that same provider type (different instances) can decrypt each other's data."""
+ # Create second local provider with the same key file
+ # First, get the key from the first provider
+ root_key = await local_file_provider.get_root_key()
+
+ # Create second provider with same key
+ with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:
+ f.write(root_key.hex())
+ temp_path2 = f.name
+ os.chmod(temp_path2, 0o600)
+
+ try:
+ provider2 = LocalFileProvider(key_file=temp_path2)
+
+ encryptor1 = FileEncryptor(local_file_provider)
+ encryptor2 = FileEncryptor(provider2)
+
+ account_id = "test-account"
+ plaintext = b"Same provider, different instance"
+
+ # Encrypt with provider1
+ ciphertext = await encryptor1.encrypt(account_id, plaintext)
+
+ # Decrypt with provider2
+ decrypted = await encryptor2.decrypt(account_id, ciphertext)
+ assert decrypted == plaintext
+ finally:
+ if os.path.exists(temp_path2):
+ os.unlink(temp_path2)