Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: add encrypt doc && refactoring encrypt code (#893)
baojun-zhang5bbb22a6ef88
diff --git a/docs/en/concepts/10-encryption.md b/docs/en/concepts/10-encryption.mdnew file mode 100644index 000000000..a00ef3122--- /dev/null+++ b/docs/en/concepts/10-encryption.md@@ -0,0 +1,220 @@+# Data Encryption++OpenViking provides transparent at-rest data encryption to ensure data security and isolation in multi-tenant environments.++## Overview++### Why Encryption++In a multi-tenant architecture, resources, memories, and skills from different customers (accounts) are stored in a shared AGFS instance. Encryption ensures:++- Even if an attacker gains AGFS disk access, they cannot read any customer's plaintext data+- Different accounts' data is encrypted with independent keys for tenant isolation+- All encryption/decryption operations are centralized at the VikingFS layer; AGFS and external object stores only see ciphertext++### Transparency++Encryption is completely transparent to users and developers:++- **No client API changes**: Existing code works without modification+- **Application layer unaware**: Read/write operations behave exactly like unencrypted+- **Backward compatible**: Unencrypted old files can still be read normally++## Three-Layer Key Architecture++OpenViking uses an Envelope Encryption architecture with a three-layer key system:++```+┌─────────────────────────────────────────────────────────┐+│ Layer 1: Root Key │+│ • Global unique per OpenViking instance │+│ • Storage: KMS service / ~/.openviking/master.key │+│ • Purpose: Derive all account keys │+└────────────────────┬────────────────────────────────────┘+ │ HKDF derivation+ ▼+┌─────────────────────────────────────────────────────────┐+│ Layer 2: Account Key (KEK) │+│ • One independent key per account │+│ • Not stored, derived at runtime │+│ • Purpose: Encrypt all file keys for this account │+└────────────────────┬────────────────────────────────────┘+ │ AES-256-GCM encryption+ ▼+┌─────────────────────────────────────────────────────────┐+│ Layer 3: File Key (DEK) │+│ • New random key generated per write operation │+│ • Stored encrypted in file header (envelope) │+│ • Purpose: Encrypt actual file content │+└─────────────────────────────────────────────────────────┘+```++### Key Hierarchy Summary++| Layer | Name | Description | Quantity |+|-------|------|-------------|----------|+| **Root Key** | Root Key | System master key, used to derive all account keys | 1 per instance |+| **Account Key** | Account Key | Independent key per account, derived from root key | 1 per account |+| **File Key** | File Key | One-time random key per file | 1 per write |++## Key Providers++OpenViking supports three key providers for different deployment scenarios:++| Provider | Use Case | Root Key Storage | Features |+|----------|----------|-----------------|----------|+| **Local** | Dev environments, single-node deployments | Local file `~/.openviking/master.key` | Simple, no external services |+| **Vault** | Production, multi-cloud | HashiCorp Vault Transit Engine | Enterprise-grade key management, version control |+| **Volcengine KMS** | Volcengine cloud deployments | Volcengine KMS | Cloud-native KMS service |++### Local (File)++Suitable for development and single-node deployments:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local",+ "local": {+ "key_file": "~/.openviking/master.key"+ }+ }+}+```++**Initialization command**:+```bash+ov crypto init-key --output ~/.openviking/master.key+```++### Vault (HashiCorp Vault)++Suitable for production and multi-cloud deployments:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "vault",+ "vault": {+ "address": "https://vault.example.com:8200",+ "token": "hvs.your-vault-token",+ "mount_point": "transit",+ "kv_mount_point": "secret",+ "kv_version": 1,+ "root_key_name": "openviking-root-key",+ "encrypted_root_key_key": "openviking-encrypted-root-key"+ }+ }+}+```++### Volcengine KMS++Suitable for Volcengine cloud deployments:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "volcengine_kms",+ "volcengine_kms": {+ "key_id": "d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx",+ "region": "cn-beijing",+ "access_key": "AKLTxxxxxxxxxxxxxxxxxx",+ "secret_key": "Tmpxxxxxxxxxxxxxxxxxxxxxx",+ "endpoint": null,+ "key_file": "~/.openviking/openviking-volcengine-root-key.enc"+ }+ }+}+```++## How It Works++### Write Flow++```+Client VikingFS FileEncryptor KeyManager AGFS+ │ │ │ │ │+ │ write(uri, data) │ │ │ │+ │──────────────────>│ │ │ │+ │ │ encrypt(account_id, │ │ │+ │ │ plaintext) │ │ │+ │ │──────────────────────>│ │ │+ │ │ │ derive_account_key()│ │+ │ │ │────────────────────>│ │+ │ │ │<────────────────────│ │+ │ │ │ account_key │ │+ │ │ 1. Generate random File Key │+ │ │ 2. Encrypt content with File Key │+ │ │ 3. Encrypt File Key with Account Key │+ │ │ 4. Build envelope format │+ │ │<──────────────────────│ │ │+ │ │ ciphertext │ │ │+ │ │──────────────────────────────────────────────────────────>│+ │ │ │ │ Write │+ │<──────────────────│ │ │ │+ │ success │ │ │ │+```++### Read Flow++```+Client VikingFS FileEncryptor KeyManager AGFS+ │ │ │ │ │+ │ read(uri) │ │ │ │+ │──────────────────>│ │ │ │+ │ │──────────────────────────────────────────────────────────>│+ │ │ │ │ Read │+ │ │<──────────────────────────────────────────────────────────│+ │ │ raw_bytes │ │ │+ │ │ Check magic == "OVE1"?│ │ │+ │ │ Yes → decrypt() │ │ │+ │ │──────────────────────>│ │ │+ │ │ │ derive_account_key()│ │+ │ │ │────────────────────>│ │+ │ │ │<────────────────────│ │+ │ │ │ account_key │ │+ │ │ 1. Parse envelope format │+ │ │ 2. Decrypt File Key with Account Key │+ │ │ 3. Decrypt content with File Key │+ │ │<──────────────────────│ │ │+ │ │ plaintext │ │ │+ │<──────────────────│ │ │ │+ │ content │ │ │ │+```++### Envelope Format++Encrypted files use a unified envelope format starting with the magic number `OVE1` (OpenViking Encryption v1):++```+┌─────────────────────────────────────────────────────────────┐+│ Magic │ Version │ Provider │ Encrypted File Key │ ... │+│ 4 bytes │ 1 byte │ 1 byte │ Variable length │ ... │+│ "OVE1" │ 0x01 │ 0x01=local│ │ ... │+└─────────────────────────────────────────────────────────────┘+```++- If a file doesn't start with `OVE1`, it's treated as unencrypted and plaintext is returned directly+- Backward compatible, old files don't need migration++## Multi-Tenant Isolation++Different accounts' data is encrypted with independent Account Keys:++- Account A's key cannot decrypt Account B's files+- Even with full AGFS access, data can't be read without the corresponding key+- Tenant isolation is implemented at the key layer, not relying on storage permissions++## Configuration Example++See [Configuration Guide](../guides/01-configuration.md#encryption) for detailed configuration.++## Related Documentation++- [Storage Architecture](./05-storage.md) - VikingFS and AGFS architecture+- [Configuration Guide](../guides/01-configuration.md) - Encryption configuration details+- [Technical Design](../../design/multi-tenant-file-encryption-desigin.md) - Encryption technical designdiff --git a/docs/en/guides/01-configuration.md b/docs/en/guides/01-configuration.mdindex 48003e28e..576ed8566 100644--- a/docs/en/guides/01-configuration.md+++ b/docs/en/guides/01-configuration.md@@ -786,6 +786,98 @@ Path locks are enabled by default and usually require no configuration. **The deFor details on the lock mechanism, see [Path Locks and Crash Recovery](../concepts/09-transaction.md).+## encryption Section++Enable at-rest data encryption to ensure data security and isolation in multi-tenant environments. Encryption is completely transparent to users with no API changes.++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local|vault|volcengine_kms"+ }+}+```++| Parameter | Type | Description | Default |+|-----------|------|-------------|---------|+| `enabled` | bool | Whether encryption is enabled | `false` |+| `provider` | str | Key provider: `"local"`, `"vault"`, or `"volcengine_kms"` | - |++### Local (File)++Suitable for development environments and single-node deployments:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local",+ "local": {+ "key_file": "~/.openviking/master.key"+ }+ }+}+```++| Parameter | Type | Description | Default |+|-----------|------|-------------|---------|+| `local.key_file` | str | Root key file path | `~/.openviking/master.key` |++### Vault (HashiCorp Vault)++Suitable for production and multi-cloud deployments:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "vault",+ "vault": {+ "address": "https://vault.example.com:8200",+ "token": "vault-token-xxx",+ "mount_point": "transit",+ "key_name": "openviking-root"+ }+ }+}+```++| Parameter | Type | Description | Default |+|-----------|------|-------------|---------|+| `vault.address` | str | Vault service address | - |+| `vault.token` | str | Vault access token | - |+| `vault.mount_point` | str | Transit engine mount point | `"transit"` |+| `vault.key_name` | str | Root key name | `"openviking-root"` |++### Volcengine KMS++Suitable for Volcengine cloud deployments:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "volcengine_kms",+ "volcengine_kms": {+ "key_id": "kms-key-id-xxx",+ "region": "cn-beijing",+ "access_key": "AKLTxxxxxxxx",+ "secret_key": "Tmpxxxxxxxx"+ }+ }+}+```++| Parameter | Type | Description | Default |+|-----------|------|-------------|---------|+| `volcengine_kms.key_id` | str | KMS key ID | - |+| `volcengine_kms.region` | str | Region | `"cn-beijing"` |+| `volcengine_kms.access_key` | str | Volcengine Access Key | - |+| `volcengine_kms.secret_key` | str | Volcengine Secret Key | - |++For detailed encryption explanations, see [Data Encryption](../concepts/10-encryption.md). For complete usage instructions, see [Encryption Guide](./08-encryption.md).+## Full Schema```json@@ -817,6 +909,25 @@ For details on the lock mechanism, see [Path Locks and Crash Recovery](../concep"api_base": "string","threshold": 0.1},+ "encryption": {+ "enabled": false,+ "provider": "local|vault|volcengine_kms",+ "local": {+ "key_file": "~/.openviking/master.key"+ },+ "vault": {+ "address": "https://vault.example.com:8200",+ "token": "string",+ "mount_point": "transit",+ "key_name": "openviking-root"+ },+ "volcengine_kms": {+ "key_id": "string",+ "region": "cn-beijing",+ "access_key": "string",+ "secret_key": "string"+ }+ },"storage": {"workspace": "string","agfs": {diff --git a/docs/en/guides/08-encryption.md b/docs/en/guides/08-encryption.mdnew file mode 100644index 000000000..e7aaf297d--- /dev/null+++ b/docs/en/guides/08-encryption.md@@ -0,0 +1,401 @@+# Encryption Guide++This guide describes how to enable and use at-rest data encryption in OpenViking.++## Overview++OpenViking provides transparent at-rest data encryption to ensure data security and isolation in multi-tenant environments:++- ✅ **Transparent encryption**: No API changes, application layer unaware+- ✅ **Multi-tenant isolation**: Different accounts use independent keys+- ✅ **Three key providers**: Local, Vault, Volcengine KMS+- ✅ **Backward compatible**: Unencrypted old files still readable++See [Data Encryption](../concepts/10-encryption.md) for conceptual explanations.++## Quick Start++### 1. Initialize Root Key (Local Mode)++```bash+ov system crypto init-key --output ~/.openviking/master.key+```++### 2. Configure Encryption++Edit `~/.openviking/ov.conf`:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local",+ "local": {+ "key_file": "~/.openviking/master.key"+ }+ },+ "storage": {+ "workspace": "./data"+ }+}+```++### 3. Verify++```python+import openviking as ov+import asyncio+++async def test():+ client = ov.AsyncOpenViking(path="./data")+ await client.initialize()++ # Add resource (automatically encrypted)+ await client.add_resource("Hello, encrypted world!", reason="Test encryption")++ # Read resource (automatically decrypted)+ results = await client.find("encrypted")+ print(f"Found {len(results)} results")++ await client.close()+++asyncio.run(test())+```++Done! Now all written data is automatically encrypted.++## Choosing a Key Provider++| Provider | Use Case | Pros | Cons |+|----------|----------|------|------|+| **Local** | Dev environments, single-node | Simple, no external services | Key stored locally, less secure |+| **Vault** | Production, multi-cloud | Enterprise-grade KMS, version control | Requires deploying and maintaining Vault |+| **Volcengine KMS** | Volcengine cloud | Cloud-native KMS service | Volcengine-only |++---++## Local Mode Detailed Guide++### Initialize Root Key++```bash+# Generate and save to specified path+ov system crypto init-key --output ~/.openviking/master.key++# Or use short option+ov system crypto init-key -o ~/.openviking/master.key+```++**Output example**:+```+✓ Root key generated successfully+✓ Saved to: /Users/you/.openviking/master.key+```++### Security Tips++- ⚠️ Keep `master.key` safe+- Recommend setting file permissions to `600` (owner-only read/write)+- Regularly back up the key file+- Don't commit the key file to version control++### Configuration Example++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local",+ "local": {+ "key_file": "~/.openviking/master.key"+ }+ }+}+```++---++## Vault Mode Detailed Guide++### Prerequisites++1. HashiCorp Vault service deployed+2. Transit engine enabled+3. Vault Token with sufficient permissions++### Configure Vault++1. Enable Transit engine (if not already enabled):++```bash+vault secrets enable transit+```++2. Enable KV engine (if not already enabled):++```bash+# KV v2 (recommended)+vault secrets enable -version=2 kv++# Or KV v1+vault secrets enable kv+```++3. Configure OpenViking:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "vault",+ "vault": {+ "address": "https://vault.example.com:8200",+ "token": "hvs.xxxxxxxxxxxxxxxxxxxxx",+ "mount_point": "transit",+ "kv_mount_point": "secret",+ "kv_version": 1,+ "root_key_name": "openviking-root-key",+ "encrypted_root_key_key": "openviking-encrypted-root-key"+ }+ }+}+```++**Configuration Parameters**:++| Parameter | Description | Default |+|-----------|-------------|---------|+| `address` | Vault server address | Required |+| `token` | Vault authentication token | Required |+| `mount_point` | Transit engine mount path | `"transit"` |+| `kv_mount_point` | KV engine mount path | `"secret"` |+| `kv_version` | KV engine version (1 or 2) | `1` |+| `root_key_name` | Key name in Transit engine | `"openviking-root-key"` |+| `encrypted_root_key_key` | Path to store encrypted root key in KV engine | `"openviking-encrypted-root-key"` |++### Vault Permission Recommendations++Configure minimal permissions for the Token:++```hcl+path "transit/encrypt/openviking-root" {+ capabilities = ["update"]+}++path "transit/decrypt/openviking-root" {+ capabilities = ["update"]+}+```++---++## Volcengine KMS Mode Detailed Guide++### Prerequisites++1. Volcengine KMS service activated+2. Symmetric key created+3. Valid Access Key and Secret Key++### Create KMS Key++1. Visit [Volcengine KMS Console](https://console.volcengine.com/kms)+2. Click "Create Key"+3. Select "Symmetric Key", algorithm `AES_256`+4. Record the Key ID++### Configure OpenViking++```json+{+ "encryption": {+ "enabled": true,+ "provider": "volcengine_kms",+ "volcengine_kms": {+ "key_id": "d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx",+ "region": "cn-beijing",+ "access_key": "AKLTxxxxxxxxxxxxxxxxxx",+ "secret_key": "Tmpxxxxxxxxxxxxxxxxxxxxxx",+ "endpoint": null,+ "key_file": "~/.openviking/openviking-volcengine-root-key.enc"+ }+ }+}+```++**Configuration Parameters**:++| Parameter | Description | Default |+|-----------|-------------|---------|+| `key_id` | KMS Key ID | Required |+| `region` | Region | Required |+| `access_key` | Access Key | Required |+| `secret_key` | Secret Key | Required |+| `endpoint` | Custom KMS endpoint (optional) | `null` (use default endpoint) |+| `key_file` | Local cache file path for encrypted root key | `"~/.openviking/openviking-volcengine-root-key.enc"` |++### Permission Recommendations++Configure minimal permissions for the Access Key:++```json+{+ "Statement": [+ {+ "Effect": "Allow",+ "Action": [+ "kms:Encrypt",+ "kms:Decrypt"+ ],+ "Resource": [+ "trn:kms:*:*:key/d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx"+ ]+ }+ ]+}+```++---++## Verifying Encryption++### Method 1: Check File Content++Encrypted files start with magic number `OVE1`:++```bash+# View first 4 bytes+hexdump -C ./data/agfs/your-file | head -1+```++**Encrypted file**:+```+00000000 4f 56 45 31 01 01 00 00 00 20 8a 7b 2c 9d 1e |OVE1..... .{,..|+```+(First 4 bytes are `4f 56 45 31` = "OVE1")++**Unencrypted file**:+```+00000000 7b 22 63 6f 6e 74 65 6e 74 73 22 3a 5b 7b 22 70 |{"contents":[{"p|+```++### Method 2: Cross-Provider Verification++Try decrypting with different providers — it should fail (this is normal security behavior):++```python+# Encrypt with Provider A+encrypted = await provider_a.encrypt_file_key(plaintext, "test-account")++# Try decrypting with Provider B (should fail)+try:+ await provider_b.decrypt_file_key(encrypted, "test-account")+ print("❌ Security vulnerability: Cross-provider decryption succeeded!")+except Exception as e:+ print("✓ Secure: Cross-provider decryption failed as expected")+```++---++## Migration Guide++### Migrating from Unencrypted to Encrypted++1. Back up existing data+2. Enable encryption (see above)+3. Re-import all resources:++```python+import openviking as ov+import asyncio+++async def migrate():+ client = ov.AsyncOpenViking(path="./data")+ await client.initialize()++ # List all resources+ resources = await client.list_resources()++ for resource in resources:+ # Read old resource (unencrypted)+ content = await client.read_resource(resource["uri"])+ # Re-write (automatically encrypted)+ await client.add_resource(content, reason="Migrate to encrypted storage")++ await client.close()+++asyncio.run(migrate())+```++### Switching Key Providers++1. Back up existing data and keys+2. Decrypt all data with old provider+3. Configure new provider+4. Re-encrypt all data++**Note**: This is a destructive operation, recommend testing first in a staging environment.++---++## Troubleshooting++### Key File Not Found++```+Error: Key file not found: ~/.openviking/master.key+```++**Solution**:+1. Check file path is correct+2. Use absolute path+3. Ensure `~` is properly expanded (use `expanduser()`)++### Vault Connection Failed++```+Error: Failed to connect to Vault+```++**Solution**:+1. Check if Vault service is running+2. Verify `address` configuration+3. Check network connectivity and firewall+4. Confirm Token is valid and not expired++### Volcengine KMS Authentication Failed++```+Error: Invalid credentials+```++**Solution**:+1. Check Access Key and Secret Key are correct+2. Confirm key has sufficient permissions+3. Verify region configuration is correct++### Cross-Provider Decryption Failed (This is Normal)++```+Error: KeyMismatchError+```++**Explanation**: This is expected security behavior. Different providers use different root keys and cannot decrypt each other's data.++### Partial Read Returns Ciphertext++If using encrypted files created with an older OpenViking version, partial reads may return ciphertext.++**Solution**: Upgrade to the latest OpenViking version.++---++## Related Documentation++- [Data Encryption](../concepts/10-encryption.md) - Encryption concepts+- [Configuration Guide](./01-configuration.md) - Complete configuration reference+- [Technical Design](../../design/multi-tenant-file-encryption-desigin.md) - Encryption technical designdiff --git a/docs/zh/concepts/10-encryption.md b/docs/zh/concepts/10-encryption.mdnew file mode 100644index 000000000..2c1b583ae--- /dev/null+++ b/docs/zh/concepts/10-encryption.md@@ -0,0 +1,220 @@+# 数据加密++OpenViking 提供透明的静态数据加密,确保多租户环境下的数据安全与隔离。++## 概述++### 为什么需要加密++在多租户架构中,不同客户(账户)的资源文件、记忆和技能都存储在共享的 AGFS 实例中。加密确保:++- 即使攻击者获得 AGFS 磁盘访问权限,也无法读取任何客户的明文数据+- 不同账户的数据使用独立密钥加密,实现租户隔离+- 所有加密解密操作集中在 VikingFS 层,AGFS 和外部对象存储只看到密文++### 对谁透明++加密功能对用户和开发者完全透明:++- **客户端 API 无变化**:现有代码无需修改+- **应用层无感知**:读写操作与未加密时完全相同+- **向后兼容**:未加密的旧文件仍可正常读取++## 三层密钥架构++OpenViking 采用信封加密(Envelope Encryption)架构,使用三层密钥体系:++```+┌─────────────────────────────────────────────────────────┐+│ Layer 1: Root Key(根密钥) │+│ • 整个 OpenViking 实例全局唯一 │+│ • 存储:KMS 服务 / ~/.openviking/master.key │+│ • 用途:派生所有账户密钥 │+└────────────────────┬────────────────────────────────────┘+ │ HKDF 派生+ ▼+┌─────────────────────────────────────────────────────────┐+│ Layer 2: Account Key(账户密钥,KEK) │+│ • 每个账户一个独立密钥 │+│ • 不存储,运行时派生 │+│ • 用途:加密该账户下的所有文件密钥 │+└────────────────────┬────────────────────────────────────┘+ │ AES-256-GCM 加密+ ▼+┌─────────────────────────────────────────────────────────┐+│ Layer 3: File Key(文件密钥,DEK) │+│ • 每次写操作生成新的随机密钥 │+│ • 加密后存储在文件头(信封)中 │+│ • 用途:加密实际文件内容 │+└─────────────────────────────────────────────────────────┘+```++### 密钥层次说明++| 层级 | 名称 | 说明 | 数量 |+|------|------|------|------|+| **Root Key** | 根密钥 | 整个系统的主密钥,用于派生所有账户密钥 | 1 个实例 |+| **Account Key** | 账户密钥 | 每个账户独立的密钥,从根密钥派生 | 每个账户 1 个 |+| **File Key** | 文件密钥 | 每个文件的一次性随机密钥 | 每次写入 1 个 |++## 密钥提供程序++OpenViking 支持三种密钥提供程序,适应不同的部署场景:++| 提供程序 | 适用场景 | Root Key 存储 | 特点 |+|---------|---------|--------------|------|+| **Local** | 开发环境、单节点部署 | 本地文件 `~/.openviking/master.key` | 简单,无需外部服务 |+| **Vault** | 生产环境、多云部署 | HashiCorp Vault Transit Engine | 企业级密钥管理,支持密钥版本控制 |+| **Volcengine KMS** | 火山引擎云部署 | 火山引擎 KMS | 云原生密钥管理服务 |++### Local(本地文件)++适合开发环境和单节点部署:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local",+ "local": {+ "key_file": "~/.openviking/master.key"+ }+ }+}+```++**初始化命令**:+```bash+ov crypto init-key --output ~/.openviking/master.key+```++### Vault(HashiCorp Vault)++适合生产环境和多云部署:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "vault",+ "vault": {+ "address": "https://vault.example.com:8200",+ "token": "hvs.your-vault-token",+ "mount_point": "transit",+ "kv_mount_point": "secret",+ "kv_version": 1,+ "root_key_name": "openviking-root-key",+ "encrypted_root_key_key": "openviking-encrypted-root-key"+ }+ }+}+```++### Volcengine KMS(火山引擎)++适合火山引擎云部署:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "volcengine_kms",+ "volcengine_kms": {+ "key_id": "d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx",+ "region": "cn-beijing",+ "access_key": "AKLTxxxxxxxxxxxxxxxxxx",+ "secret_key": "Tmpxxxxxxxxxxxxxxxxxxxxxx",+ "endpoint": null,+ "key_file": "~/.openviking/openviking-volcengine-root-key.enc"+ }+ }+}+```++## 工作原理++### 写流程++```+客户端 VikingFS FileEncryptor KeyManager AGFS+ │ │ │ │ │+ │ write(uri, data) │ │ │ │+ │──────────────────>│ │ │ │+ │ │ encrypt(account_id, │ │ │+ │ │ plaintext) │ │ │+ │ │──────────────────────>│ │ │+ │ │ │derive_account_key()│ │+ │ │ │───────────────────>│ │+ │ │ │<───────────────────│ │+ │ │ │ account_key │ │+ │ │ 1. 生成随机 File Key │ │+ │ │ 2. 用 File Key 加密内容 │ │+ │ │ 3. 用 Account Key 加密 File Key │ │+ │ │ 4. 构建信封格式 │ │+ │ │<──────────────────────│ │ │+ │ │ ciphertext │ │ │+ │ │─────────────────────────────────────────────────────────>│+ │ │ │ │ Write │+ │<──────────────────│ │ │ │+ │ success │ │ │ │+```++### 读流程++```+客户端 VikingFS FileEncryptor KeyManager AGFS+ │ │ │ │ │+ │ read(uri) │ │ │ │+ │──────────────────>│ │ │ │+ │ │──────────────────────────────────────────────────────────>│+ │ │ │ │ Read │+ │ │<──────────────────────────────────────────────────────────│+ │ │ raw_bytes │ │ │+ │ │ 检查魔术数 == "OVE1"? │ │ │+ │ │ 是 → decrypt() │ │ │+ │ │──────────────────────>│ │ │+ │ │ │ derive_account_key()│ │+ │ │ │────────────────────>│ │+ │ │ │<────────────────────│ │+ │ │ │ account_key │ │+ │ │ 1. 解析信封格式 │ │+ │ │ 2. 用 Account Key 解密 File Key │ │+ │ │ 3. 用 File Key 解密内容 │ │+ │ │<──────────────────────│ │ │+ │ │ plaintext │ │ │+ │<──────────────────│ │ │ │+ │ content │ │ │ │+```++### 信封格式++加密文件使用统一的信封格式,以魔术数 `OVE1`(OpenViking Encryption v1)开头:++```+┌─────────────────────────────────────────────────────────────┐+│ 魔术数 │ 版本 │ Provider │ 加密的 File Key │ .. │+│ 4 字节 │ 1 字节 │ 1 字节 │ 可变长度 │ ... │+│ "OVE1" │ 0x01 │ 0x01=local │ │ ... │+└─────────────────────────────────────────────────────────────┘+```++- 如果文件不以 `OVE1` 开头,视为未加密文件,直接返回明文+- 支持向后兼容,旧文件无需迁移++## 多租户隔离++不同账户的数据使用独立的 Account Key 加密:++- 账户 A 的密钥无法解密账户 B 的文件+- 即使 AGFS 被完全访问,没有对应密钥也无法读取数据+- 租户隔离在密钥层面实现,不依赖存储层权限++## 配置示例++详细配置说明请参考 [配置文档](../guides/01-configuration.md#encryption)。++## 相关文档++- [存储架构](./05-storage.md) - VikingFS 和 AGFS 架构+- [配置指南](../guides/01-configuration.md) - 加密配置详解+- [技术设计](../../design/multi-tenant-file-encryption-desigin.md) - 加密技术设计文档diff --git a/docs/zh/guides/01-configuration.md b/docs/zh/guides/01-configuration.mdindex a5c3507df..83d05327c 100644--- a/docs/zh/guides/01-configuration.md+++ b/docs/zh/guides/01-configuration.md@@ -741,6 +741,98 @@ HTTP 客户端(`SyncHTTPClient` / `AsyncHTTPClient`)和 CLI 工具连接远启动方式和部署详情见 [服务部署](./03-deployment.md),认证详情见 [认证](./04-authentication.md)。+## encryption 段++启用静态数据加密,确保多租户环境下的数据安全与隔离。加密功能对用户完全透明,API 无变化。++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local|vault|volcengine_kms"+ }+}+```++| 参数 | 类型 | 说明 | 默认值 |+|------|------|------|--------|+| `enabled` | bool | 是否启用加密 | `false` |+| `provider` | str | 密钥提供程序:`"local"`、`"vault"` 或 `"volcengine_kms"` | - |++### Local(本地文件)++适合开发环境和单节点部署:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local",+ "local": {+ "key_file": "~/.openviking/master.key"+ }+ }+}+```++| 参数 | 类型 | 说明 | 默认值 |+|------|------|------|--------|+| `local.key_file` | str | 根密钥文件路径 | `~/.openviking/master.key` |++### Vault(HashiCorp Vault)++适合生产环境和多云部署:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "vault",+ "vault": {+ "address": "https://vault.example.com:8200",+ "token": "vault-token-xxx",+ "mount_point": "transit",+ "key_name": "openviking-root"+ }+ }+}+```++| 参数 | 类型 | 说明 | 默认值 |+|------|------|------|--------|+| `vault.address` | str | Vault 服务地址 | - |+| `vault.token` | str | Vault 访问令牌 | - |+| `vault.mount_point` | str | Transit 引擎挂载点 | `"transit"` |+| `vault.key_name` | str | 根密钥名称 | `"openviking-root"` |++### Volcengine KMS(火山引擎)++适合火山引擎云部署:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "volcengine_kms",+ "volcengine_kms": {+ "key_id": "kms-key-id-xxx",+ "region": "cn-beijing",+ "access_key": "AKLTxxxxxxxx",+ "secret_key": "Tmpxxxxxxxx"+ }+ }+}+```++| 参数 | 类型 | 说明 | 默认值 |+|------|------|------|--------|+| `volcengine_kms.key_id` | str | KMS 密钥 ID | - |+| `volcengine_kms.region` | str | 区域 | `"cn-beijing"` |+| `volcengine_kms.access_key` | str | 火山引擎 Access Key | - |+| `volcengine_kms.secret_key` | str | 火山引擎 Secret Key | - |++加密功能的详细说明见 [数据加密](../concepts/10-encryption.md),完整使用流程见 [加密指南](./08-encryption.md)。+## storage.transaction 段路径锁默认启用,通常无需配置。**默认行为是不等待**:若目标路径已被其他操作锁定,操作立即失败并抛出 `LockAcquisitionError`。若需要等待重试,请将 `lock_timeout` 设为正数。@@ -794,6 +886,25 @@ HTTP 客户端(`SyncHTTPClient` / `AsyncHTTPClient`)和 CLI 工具连接远"api_base": "string","threshold": 0.1},+ "encryption": {+ "enabled": false,+ "provider": "local|vault|volcengine_kms",+ "local": {+ "key_file": "~/.openviking/master.key"+ },+ "vault": {+ "address": "https://vault.example.com:8200",+ "token": "string",+ "mount_point": "transit",+ "key_name": "openviking-root"+ },+ "volcengine_kms": {+ "key_id": "string",+ "region": "cn-beijing",+ "access_key": "string",+ "secret_key": "string"+ }+ },"storage": {"workspace": "string","agfs": {diff --git a/docs/zh/guides/08-encryption.md b/docs/zh/guides/08-encryption.mdnew file mode 100644index 000000000..9e8d60fb5--- /dev/null+++ b/docs/zh/guides/08-encryption.md@@ -0,0 +1,401 @@+# 加密指南++本指南介绍如何在 OpenViking 中启用和使用静态数据加密功能。++## 概述++OpenViking 提供透明的静态数据加密,确保多租户环境下的数据安全与隔离:++- ✅ **透明加密**:API 无变化,应用层无感知+- ✅ **多租户隔离**:不同账户使用独立密钥+- ✅ **三种密钥提供程序**:Local、Vault、火山引擎 KMS+- ✅ **向后兼容**:未加密的旧文件仍可正常读取++加密功能的概念说明见 [数据加密](../concepts/10-encryption.md)。++## 快速开始++### 1. 初始化根密钥(Local 模式)++```bash+ov system crypto init-key --output ~/.openviking/master.key+```++### 2. 配置加密++编辑 `~/.openviking/ov.conf`:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local",+ "local": {+ "key_file": "~/.openviking/master.key"+ }+ },+ "storage": {+ "workspace": "./data"+ }+}+```++### 3. 验证++```python+import openviking as ov+import asyncio+++async def test():+ client = ov.AsyncOpenViking(path="./data")+ await client.initialize()++ # 添加资源(自动加密)+ await client.add_resource("Hello, encrypted world!", reason="测试加密")++ # 读取资源(自动解密)+ results = await client.find("encrypted")+ print(f"找到 {len(results)} 个结果")++ await client.close()+++asyncio.run(test())+```++完成!现在所有写入的数据都会自动加密。++## 密钥提供程序选择++| 提供程序 | 适用场景 | 优点 | 缺点 |+|---------|---------|------|------|+| **Local** | 开发环境、单节点部署 | 简单,无需外部服务 | 密钥存储在本地,安全性较低 |+| **Vault** | 生产环境、多云部署 | 企业级密钥管理,支持版本控制 | 需要部署和维护 Vault |+| **Volcengine KMS** | 火山引擎云部署 | 云原生密钥管理服务 | 仅限火山引擎环境 |++---++## Local 模式详细指南++### 初始化根密钥++```bash+# 生成并保存到指定路径+ov system crypto init-key --output ~/.openviking/master.key++# 或者使用简短命令+ov system crypto init-key -o ~/.openviking/master.key+```++**输出示例**:+```+✓ Root key generated successfully+✓ Saved to: /Users/you/.openviking/master.key+```++### 安全提示++- ⚠️ 妥善保管 `master.key` 文件+- 建议设置文件权限为 `600`(仅所有者可读写)+- 定期备份密钥文件+- 不要将密钥文件提交到版本控制系统++### 配置示例++```json+{+ "encryption": {+ "enabled": true,+ "provider": "local",+ "local": {+ "key_file": "~/.openviking/master.key"+ }+ }+}+```++---++## Vault 模式详细指南++### 前置条件++1. 已部署 HashiCorp Vault 服务+2. 已启用 Transit 引擎+3. 有足够权限的 Vault Token++### 配置 Vault++1. 启用 Transit 引擎(如果尚未启用):++```bash+vault secrets enable transit+```++2. 启用 KV 引擎(如果尚未启用):++```bash+# KV v2(推荐)+vault secrets enable -version=2 kv++# 或 KV v1+vault secrets enable kv+```++3. 配置 OpenViking:++```json+{+ "encryption": {+ "enabled": true,+ "provider": "vault",+ "vault": {+ "address": "https://vault.example.com:8200",+ "token": "hvs.xxxxxxxxxxxxxxxxxxxxx",+ "mount_point": "transit",+ "kv_mount_point": "secret",+ "kv_version": 1,+ "root_key_name": "openviking-root-key",+ "encrypted_root_key_key": "openviking-encrypted-root-key"+ }+ }+}+```++**配置参数说明**:++| 参数 | 说明 | 默认值 |+|------|------|--------|+| `address` | Vault 服务器地址 | 必需 |+| `token` | Vault 认证令牌 | 必需 |+| `mount_point` | Transit 引擎挂载路径 | `"transit"` |+| `kv_mount_point` | KV 引擎挂载路径 | `"secret"` |+| `kv_version` | KV 引擎版本(1 或 2) | `1` |+| `root_key_name` | Transit 引擎中的密钥名称 | `"openviking-root-key"` |+| `encrypted_root_key_key` | KV 引擎中存储加密根密钥的路径 | `"openviking-encrypted-root-key"` |++### Vault 权限建议++为 Token 配置最小权限:++```hcl+path "transit/encrypt/openviking-root" {+ capabilities = ["update"]+}++path "transit/decrypt/openviking-root" {+ capabilities = ["update"]+}+```++---++## Volcengine KMS 模式详细指南++### 前置条件++1. 已开通火山引擎 KMS 服务+2. 已创建对称密钥+3. 有有效的 Access Key 和 Secret Key++### 创建 KMS 密钥++1. 访问 [火山引擎 KMS 控制台](https://console.volcengine.com/kms)+2. 点击"创建密钥"+3. 选择"对称密钥",算法选择 `AES_256`+4. 记录密钥 ID++### 配置 OpenViking++```json+{+ "encryption": {+ "enabled": true,+ "provider": "volcengine_kms",+ "volcengine_kms": {+ "key_id": "d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx",+ "region": "cn-beijing",+ "access_key": "AKLTxxxxxxxxxxxxxxxxxx",+ "secret_key": "Tmpxxxxxxxxxxxxxxxxxxxxxx",+ "endpoint": null,+ "key_file": "~/.openviking/openviking-volcengine-root-key.enc"+ }+ }+}+```++**配置参数说明**:++| 参数 | 说明 | 默认值 |+|------|------|--------|+| `key_id` | KMS 密钥 ID | 必需 |+| `region` | 区域 | 必需 |+| `access_key` | Access Key | 必需 |+| `secret_key` | Secret Key | 必需 |+| `endpoint` | 自定义 KMS 端点(可选) | `null`(使用默认端点) |+| `key_file` | 加密根密钥本地缓存文件路径 | `"~/.openviking/openviking-volcengine-root-key.enc"` |++### 权限建议++为 Access Key 配置最小权限:++```json+{+ "Statement": [+ {+ "Effect": "Allow",+ "Action": [+ "kms:Encrypt",+ "kms:Decrypt"+ ],+ "Resource": [+ "trn:kms:*:*:key/d926aa0d-xxxx-xxxx-xxxx-xxxxxxxxxxxx"+ ]+ }+ ]+}+```++---++## 验证加密++### 方法一:检查文件内容++加密文件以魔术数 `OVE1` 开头:++```bash+# 查看文件前 4 字节+hexdump -C ./data/agfs/your-file | head -1+```++**加密文件**:+```+00000000 4f 56 45 31 01 01 00 00 00 20 8a 7b 2c 9d 1e |OVE1..... .{,..|+```+(前 4 字节是 `4f 56 45 31` = "OVE1")++**未加密文件**:+```+00000000 7b 22 63 6f 6e 74 65 6e 74 73 22 3a 5b 7b 22 70 |{"contents":[{"p|+```++### 方法二:跨提供程序验证++尝试用不同提供程序解密彼此的数据,应该会失败(这是正常的安全行为):++```python+# 用 Provider A 加密+encrypted = await provider_a.encrypt_file_key(plaintext, "test-account")++# 尝试用 Provider B 解密(应该失败)+try:+ await provider_b.decrypt_file_key(encrypted, "test-account")+ print("❌ 安全漏洞:跨提供程序解密成功!")+except Exception as e:+ print("✓ 安全:跨提供程序解密失败,符合预期")+```++---++## 迁移说明++### 从无加密迁移到有加密++1. 备份现有数据+2. 启用加密(参考上文)+3. 重新导入所有资源:++```python+import openviking as ov+import asyncio+++async def migrate():+ client = ov.AsyncOpenViking(path="./data")+ await client.initialize()++ # 列出所有资源+ resources = await client.list_resources()++ for resource in resources:+ # 读取旧资源(未加密)+ content = await client.read_resource(resource["uri"])+ # 重新写入(自动加密)+ await client.add_resource(content, reason="迁移到加密存储")++ await client.close()+++asyncio.run(migrate())+```++### 切换密钥提供程序++1. 备份现有数据和密钥+2. 使用旧提供程序解密所有数据+3. 配置新提供程序+4. 重新加密所有数据++**注意**:这是一个破坏性操作,建议在测试环境先验证。++---++## 故障排除++### 密钥文件找不到++```+Error: Key file not found: ~/.openviking/master.key+```++**解决方案**:+1. 检查文件路径是否正确+2. 使用绝对路径+3. 确保 `~` 被正确展开(使用 `expanduser()`)++### Vault 连接失败++```+Error: Failed to connect to Vault+```++**解决方案**:+1. 检查 Vault 服务是否运行+2. 验证 `address` 配置+3. 检查网络连接和防火墙+4. 确认 Token 有效且未过期++### 火山 KMS 认证失败++```+Error: Invalid credentials+```++**解决方案**:+1. 检查 Access Key 和 Secret Key 是否正确+2. 确认密钥有足够权限+3. 验证区域配置正确++### 跨提供程序解密失败(这是正常的)++```+Error: KeyMismatchError+```++**说明**:这是预期的安全行为。不同提供程序使用不同的根密钥,无法相互解密。++### 部分读取返回密文++如果使用旧版本 OpenViking 创建的加密文件,部分读取可能返回密文。++**解决方案**:升级到最新版本的 OpenViking。++---++## 相关文档++- [数据加密](../concepts/10-encryption.md) - 加密概念说明+- [配置指南](./01-configuration.md) - 完整配置参考+- [技术设计](../../design/multi-tenant-file-encryption-desigin.md) - 加密技术设计文档diff --git a/examples/ov.conf.example b/examples/ov.conf.exampleindex c4211385c..5b837b8db 100644--- a/examples/ov.conf.example+++ b/examples/ov.conf.example@@ -161,12 +161,39 @@"max_paragraph_length": 1000,"preserve_line_breaks": false}- }+ },"encryption": {"enabled": true,"provider": "local","local": {"key_file": "~/.openviking/master.key"}+ },+ "encryption_vault_example": {+ "_comment": "For HashiCorp Vault integration:",+ "enabled": true,+ "provider": "vault",+ "vault": {+ "address": "http://127.0.0.1:8200",+ "token": "hvs.your-vault-token",+ "mount_point": "transit",+ "kv_mount_point": "secret",+ "kv_version": 1,+ "root_key_name": "openviking-root-key",+ "encrypted_root_key_key": "openviking-encrypted-root-key"+ }+ },+ "encryption_volcengine_kms_example": {+ "_comment": "For Volcengine KMS integration:",+ "enabled": true,+ "provider": "volcengine_kms",+ "volcengine_kms": {+ "region": "cn-beijing",+ "access_key": "AKLTyour-access-key",+ "secret_key": "your-secret-key",+ "key_id": "your-kms-key-id",+ "endpoint": null,+ "key_file": "~/.openviking/openviking-volcengine-root-key.enc"+ }}}diff --git a/openviking/crypto/config.py b/openviking/crypto/config.pyindex bba2067bc..109e8190f 100644--- a/openviking/crypto/config.py+++ b/openviking/crypto/config.py@@ -67,22 +67,25 @@ def _validate_local_provider_config(config: Dict[str, Any]) -> List[str]:return errors# Check if file exists or can be created- key_file = Path(key_file_path)+ key_file = Path(key_file_path).expanduser()if key_file.exists():# Check permissionsif os.name != "nt": # Skip permission check on Windowsif (key_file.stat().st_mode & 0o077) != 0:errors.append(f"Key file permissions too open: {key_file_path} (should be 0600)")else:- # Check if parent directory is writable+ # Check if parent directory exists or can be written toparent_dir = key_file.parent- if not parent_dir.exists():- try:- parent_dir.mkdir(parents=True, exist_ok=True)- except Exception as e:- errors.append(f"Cannot create parent directory for key file: {e}")- elif not os.access(parent_dir, os.W_OK):- errors.append(f"Cannot create key file at: {key_file_path}")+ if parent_dir.exists():+ if not os.access(parent_dir, os.W_OK):+ errors.append(f"Cannot create key file at: {key_file_path}")+ else:+ # Check if we can write to the closest existing ancestor directory+ current = parent_dir+ while not current.exists() and current.parent != current:+ current = current.parent+ if not os.access(current, os.W_OK):+ errors.append(f"Cannot create parent directory for key file at: {parent_dir}")return errorsdiff --git a/openviking/crypto/encryptor.py b/openviking/crypto/encryptor.pyindex 82943485b..953c2d1d6 100644--- a/openviking/crypto/encryptor.py+++ b/openviking/crypto/encryptor.py@@ -85,12 +85,8 @@ class FileEncryptor:data_iv = secrets.token_bytes(12)encrypted_content = await self._aes_gcm_encrypt(file_key, data_iv, plaintext)- # 3. Encrypt File Key- key_iv = b""- if self._provider_type == PROVIDER_LOCAL:- encrypted_file_key, key_iv = await self.provider.encrypt_file_key(file_key, account_id)- else:- encrypted_file_key = await self.provider.encrypt_file_key(file_key, account_id)+ # 3. Encrypt File Key (all providers now return (encrypted_key, iv)+ encrypted_file_key, key_iv = await self.provider.encrypt_file_key(file_key, account_id)# 4. Build Envelopereturn self._build_envelope(@@ -133,13 +129,8 @@ class FileEncryptor:raise CorruptedCiphertextError(f"Failed to parse envelope: {e}")try:- # 3. Decrypt File Key- if provider_type == PROVIDER_LOCAL:- file_key = await self.provider.decrypt_file_key(- encrypted_file_key, key_iv, account_id- )- else:- file_key = await self.provider.decrypt_file_key(encrypted_file_key, account_id)+ # 3. Decrypt File Key (all providers now use (encrypted_key, iv, account_id)+ file_key = await self.provider.decrypt_file_key(encrypted_file_key, key_iv, account_id)except Exception as e:raise KeyMismatchError(f"Failed to decrypt file key: {e}")diff --git a/openviking/crypto/providers.py b/openviking/crypto/providers.pyindex 9a29f7318..3f8bf68a8 100644--- a/openviking/crypto/providers.py+++ b/openviking/crypto/providers.py@@ -10,6 +10,7 @@ Provides multiple key management methods:"""import abc+import asyncioimport osimport secretsfrom abc import ABC@@ -48,17 +49,103 @@ class RootKeyProvider(ABC):pass@abc.abstractmethod- async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> Any:+ async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> Tuple[bytes, bytes]:"""Encrypt File Key."""pass@abc.abstractmethod- async def decrypt_file_key(self, encrypted_key: Any, account_id: str) -> bytes:+ async def decrypt_file_key(self, encrypted_key: bytes, iv: bytes, account_id: str) -> bytes:"""Decrypt File Key."""pass+ async def _hkdf_derive(+ self, root_key: bytes, account_id: str, salt: bytes, info_prefix: bytes+ ) -> bytes:+ """+ Derive key using HKDF.++ Args:+ root_key: Root key+ account_id: Account ID+ salt: HKDF salt+ info_prefix: HKDF info prefix++ Returns:+ Derived key+ """+ try:+ from cryptography.hazmat.primitives import hashes+ from cryptography.hazmat.primitives.kdf.hkdf import HKDF++ hkdf = HKDF(+ algorithm=hashes.SHA256(),+ length=32,+ salt=salt,+ info=info_prefix + account_id.encode(),+ )+ return hkdf.derive(root_key)+ except ImportError:+ raise ConfigError("cryptography library is required for encryption")+++class BaseProvider(RootKeyProvider):+ """Base provider with common encryption functionality."""++ async def _aes_gcm_encrypt(self, key: bytes, iv: bytes, plaintext: bytes) -> bytes:+ """AES-GCM encryption."""+ try:+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM++ aesgcm = AESGCM(key)+ return aesgcm.encrypt(iv, plaintext, associated_data=None)+ except ImportError:+ raise ConfigError("cryptography library is required for encryption")++ async def _aes_gcm_decrypt(self, key: bytes, iv: bytes, ciphertext: bytes) -> bytes:+ """AES-GCM decryption."""+ try:+ from cryptography.hazmat.primitives.ciphers.aead import AESGCM++ aesgcm = AESGCM(key)+ return aesgcm.decrypt(iv, ciphertext, associated_data=None)+ except ImportError:+ raise ConfigError("cryptography library is required for encryption")+ except Exception as e:+ raise AuthenticationFailedError(f"Decryption failed: {e}")++ async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> Tuple[bytes, bytes]:+ """+ Encrypt File Key with Account Key.-class LocalFileProvider(RootKeyProvider):+ Args:+ plaintext_key: Plaintext File Key+ account_id: Account ID++ Returns:+ (encrypted_key, iv)+ """+ account_key = await self.derive_account_key(account_id)+ iv = secrets.token_bytes(12)+ encrypted_key = await self._aes_gcm_encrypt(account_key, iv, plaintext_key)+ return encrypted_key, iv++ async def decrypt_file_key(self, encrypted_key: bytes, iv: bytes, account_id: str) -> bytes:+ """+ Decrypt File Key with Account Key.++ Args:+ encrypted_key: Encrypted File Key+ iv: Initialization vector+ account_id: Account ID++ Returns:+ Decrypted File Key+ """+ account_key = await self.derive_account_key(account_id)+ return await self._aes_gcm_decrypt(account_key, iv, encrypted_key)+++class LocalFileProvider(BaseProvider):"""Local file Root Key Provider."""def __init__(self, key_file: str):@@ -68,7 +155,7 @@ class LocalFileProvider(RootKeyProvider):Args:key_file: Root Key file path"""- self.key_file = Path(key_file)+ self.key_file = Path(key_file).expanduser()self._root_key: Optional[bytes] = Noneasync def get_root_key(self) -> bytes:@@ -103,79 +190,29 @@ class LocalFileProvider(RootKeyProvider):async def derive_account_key(self, account_id: str) -> bytes:"""Derive Account Key from Root Key."""root_key = await self.get_root_key()- return await self._hkdf_derive(root_key, account_id)+ return await self._hkdf_derive(root_key, account_id, HKDF_SALT, HKDF_INFO_PREFIX)- async def _hkdf_derive(self, root_key: bytes, account_id: str) -> bytes:- """Derive key using HKDF."""- try:- from cryptography.hazmat.primitives import hashes- from cryptography.hazmat.primitives.kdf.hkdf import HKDF- hkdf = HKDF(- algorithm=hashes.SHA256(),- length=32,- salt=HKDF_SALT,- info=HKDF_INFO_PREFIX + account_id.encode(),- )- return hkdf.derive(root_key)- except ImportError:- raise ConfigError("cryptography library is required for encryption")-- async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> Tuple[bytes, bytes]:- """- Encrypt File Key.-- Returns:- (encrypted_key, iv)- """- account_key = await self.derive_account_key(account_id)- iv = secrets.token_bytes(12)- encrypted_key = await self._aes_gcm_encrypt(account_key, iv, plaintext_key)- return encrypted_key, iv-- async def decrypt_file_key(self, encrypted_key: bytes, iv: bytes, account_id: str) -> bytes:- """Decrypt File Key."""- account_key = await self.derive_account_key(account_id)- return await self._aes_gcm_decrypt(account_key, iv, encrypted_key)-- async def _aes_gcm_encrypt(self, key: bytes, iv: bytes, plaintext: bytes) -> bytes:- """AES-GCM encryption."""- try:- from cryptography.hazmat.primitives.ciphers.aead import AESGCM-- aesgcm = AESGCM(key)- return aesgcm.encrypt(iv, plaintext, associated_data=None)- except ImportError:- raise ConfigError("cryptography library is required for encryption")-- async def _aes_gcm_decrypt(self, key: bytes, iv: bytes, ciphertext: bytes) -> bytes:- """AES-GCM decryption."""- try:- from cryptography.hazmat.primitives.ciphers.aead import AESGCM-- aesgcm = AESGCM(key)- return aesgcm.decrypt(iv, ciphertext, associated_data=None)- except ImportError:- raise ConfigError("cryptography library is required for encryption")- except Exception as e:- raise AuthenticationFailedError(f"Decryption failed: {e}")---class VaultProvider(RootKeyProvider):+class VaultProvider(BaseProvider):"""HashiCorp Vault Key Provider.Uses HashiCorp Vault's transit secrets engine for key management.Core features:- - Root key management: Vault transit secrets engine+ - Root key management: Vault transit secrets engine (encrypted)- Account Key derivation: HKDF-SHA256 (from root key)- - File Key encryption: Vault transit encryption API+ - File Key encryption: AES-256-GCM (with Account Key)"""- SALT = b"OpenViking_KDF_Salt"- INFO_PREFIX = b"OpenViking_Account_"- ROOT_KEY_NAME = "openviking-root-key"-- def __init__(self, addr: str, token: str, mount_path: str = "transit"):+ def __init__(+ self,+ addr: str,+ token: str,+ mount_path: str = "transit",+ kv_mount_path: str = "secret",+ kv_version: int = 1,+ root_key_name: str = "openviking-root-key",+ encrypted_root_key_key: str = "openviking-encrypted-root-key",+ ):"""Initialize VaultProvider.@@ -183,10 +220,18 @@ class VaultProvider(RootKeyProvider):addr: Vault server address (e.g., "http://127.0.0.1:8200")token: Vault authentication tokenmount_path: Transit secrets engine mount path (default: "transit")+ kv_mount_path: KV secrets engine mount path (default: "secret")+ kv_version: KV secrets engine version (1 or 2, default: 1)+ root_key_name: Transit engine key name (default: "openviking-root-key")+ encrypted_root_key_key: KV engine key path (default: "openviking-encrypted-root-key")"""self.addr = addrself.token = tokenself.mount_path = mount_path+ self.kv_mount_path = kv_mount_path+ self.kv_version = kv_version+ self.root_key_name = root_key_name+ self.encrypted_root_key_key = encrypted_root_key_keyself._client = Noneself._root_key: Optional[bytes] = None@@ -208,7 +253,8 @@ class VaultProvider(RootKeyProvider):self._client = hvac.Client(url=self.addr, token=self.token)# Verify Vault is accessible- if not self._client.is_authenticated():+ is_auth = await asyncio.to_thread(self._client.is_authenticated)+ if not is_auth:raise AuthenticationFailedError("Failed to authenticate with Vault")# Ensure transit engine is enabled@@ -223,9 +269,13 @@ class VaultProvider(RootKeyProvider):"""Ensure transit secrets engine is enabled."""try:# Check if transit engine is already enabled- self._client.sys.list_mounted_secrets_engines()- if f"{self.mount_path}/" not in self._client.sys.list_mounted_secrets_engines()["data"]:- self._client.sys.enable_secrets_engine(backend_type="transit", path=self.mount_path)+ engines = await asyncio.to_thread(self._client.sys.list_mounted_secrets_engines)+ if f"{self.mount_path}/" not in engines["data"]:+ await asyncio.to_thread(+ self._client.sys.enable_secrets_engine,+ backend_type="transit",+ path=self.mount_path,+ )logger.info(f"Enabled transit secrets engine at {self.mount_path}")except Exception as e:logger.warning(f"Failed to check/enable transit engine: {e}")@@ -234,129 +284,173 @@ class VaultProvider(RootKeyProvider):"""Ensure root key exists in Vault transit engine."""try:# Try to read the key- self._client.secrets.transit.read_key(- name=self.ROOT_KEY_NAME, mount_point=self.mount_path+ await asyncio.to_thread(+ self._client.secrets.transit.read_key,+ name=self.root_key_name,+ mount_point=self.mount_path,)except Exception:# Key doesn't exist, create it- self._client.secrets.transit.create_key(- name=self.ROOT_KEY_NAME, key_type="aes256-gcm96", mount_point=self.mount_path+ await asyncio.to_thread(+ self._client.secrets.transit.create_key,+ name=self.root_key_name,+ key_type="aes256-gcm96",+ mount_point=self.mount_path,)- logger.info(f"Created root key {self.ROOT_KEY_NAME} in Vault")+ logger.info(f"Created root key {self.root_key_name} in Vault")- async def get_root_key(self) -> bytes:+ async def _encrypt_with_vault(self, plaintext: bytes) -> bytes:"""- Get root key from Vault.-- Note: Since we can't export the key directly from Vault transit,- we use a fixed seed and derive the root key.-- Returns:- Root key- """- if self._root_key is None:- # Use a fixed seed for root key derivation- self._root_key = b"OpenViking_Vault_Root_Seed_Key_v1"- return self._root_key-- async def derive_account_key(self, account_id: str) -> bytes:- """- Derive Account Key using HKDF.+ Encrypt data with Vault transit.Args:- account_id: Account ID+ plaintext: Plaintext dataReturns:- Derived Account Key+ Encrypted data"""- root_key = await self.get_root_key()- return await self._hkdf_derive(root_key, account_id)+ client = await self._get_client()+ import base64- async def _hkdf_derive(self, root_key: bytes, account_id: str) -> bytes:+ plaintext_b64 = base64.b64encode(plaintext).decode("utf-8")+ response = await asyncio.to_thread(+ client.secrets.transit.encrypt_data,+ name=self.root_key_name,+ plaintext=plaintext_b64,+ mount_point=self.mount_path,+ )+ ciphertext_str = response["data"]["ciphertext"]+ return ciphertext_str.encode("utf-8")++ async def _decrypt_with_vault(self, ciphertext: bytes) -> bytes:"""- Derive key using HKDF.+ Decrypt data with Vault transit.Args:- root_key: Root key- account_id: Account ID+ ciphertext: Encrypted dataReturns:- Derived key+ Decrypted data"""- try:- from cryptography.hazmat.primitives import hashes- from cryptography.hazmat.primitives.kdf.hkdf import HKDF+ client = await self._get_client()+ import base64- hkdf = HKDF(- algorithm=hashes.SHA256(),- length=32,- salt=self.SALT,- info=self.INFO_PREFIX + account_id.encode(),- )- return hkdf.derive(root_key)- except ImportError:- raise ConfigError("cryptography library is required for encryption")+ ciphertext_str = ciphertext.decode("utf-8")+ response = await asyncio.to_thread(+ client.secrets.transit.decrypt_data,+ name=self.root_key_name,+ ciphertext=ciphertext_str,+ mount_point=self.mount_path,+ )+ return base64.b64decode(response["data"]["plaintext"])- async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> bytes:+ async def _get_or_create_root_key(self) -> bytes:"""- Encrypt File Key using Vault transit.-- Args:- plaintext_key: Plaintext File Key- account_id: Account ID (not used in Vault mode)+ Get or create root key.Returns:- Encrypted File Key as bytes+ Root key bytes"""- client = await self._get_client()+ if self._root_key is not None:+ return self._root_key+ client = await self._get_client()import base64- plaintext_b64 = base64.b64encode(plaintext_key).decode("utf-8")+ try:+ # Try to read encrypted root key from Vault kv+ if self.kv_version == 2:+ response = await asyncio.to_thread(+ client.secrets.kv.v2.read_secret_version,+ path=self.encrypted_root_key_key,+ mount_point=self.kv_mount_path,+ )+ encrypted_root_key_b64 = response["data"]["data"]["encrypted_root_key"]+ else:+ response = await asyncio.to_thread(+ client.secrets.kv.v1.read_secret,+ path=self.encrypted_root_key_key,+ mount_point=self.kv_mount_path,+ )+ encrypted_root_key_b64 = response["data"]["encrypted_root_key"]- response = client.secrets.transit.encrypt_data(- name=self.ROOT_KEY_NAME, plaintext=plaintext_b64, mount_point=self.mount_path- )+ encrypted_root_key = base64.b64decode(encrypted_root_key_b64)+ self._root_key = await self._decrypt_with_vault(encrypted_root_key)+ logger.info("Loaded existing root key from Vault")+ except Exception:+ # Generate new root key+ import secrets- ciphertext_str = response["data"]["ciphertext"]- return ciphertext_str.encode("utf-8")+ self._root_key = secrets.token_bytes(32)- async def decrypt_file_key(self, encrypted_key: bytes, account_id: str) -> bytes:- """- Decrypt File Key using Vault transit.+ # Encrypt and store root key in Vault kv+ encrypted_root_key = await self._encrypt_with_vault(self._root_key)+ try:+ if self.kv_version == 2:+ await asyncio.to_thread(+ client.secrets.kv.v2.create_or_update_secret,+ path=self.encrypted_root_key_key,+ mount_point=self.kv_mount_path,+ secret={+ "encrypted_root_key": base64.b64encode(encrypted_root_key).decode(+ "utf-8"+ )+ },+ )+ else:+ await asyncio.to_thread(+ client.secrets.kv.v1.create_or_update_secret,+ path=self.encrypted_root_key_key,+ mount_point=self.kv_mount_path,+ secret={+ "encrypted_root_key": base64.b64encode(encrypted_root_key).decode(+ "utf-8"+ )+ },+ )+ logger.info("Created and stored new root key in Vault")+ except Exception as e:+ raise ConfigError(+ f"Failed to persist root key to Vault. "+ f"Refusing to start with ephemeral key (data loss risk): {e}"+ )- Args:- encrypted_key: Encrypted File Key (ciphertext bytes from Vault)- account_id: Account ID (not used in Vault mode)+ return self._root_key- Returns:- Decrypted File Key+ async def get_root_key(self) -> bytes:"""- client = await self._get_client()+ Get root key.- ciphertext_str = encrypted_key.decode("utf-8")+ Returns:+ Root key+ """+ return await self._get_or_create_root_key()- response = client.secrets.transit.decrypt_data(- name=self.ROOT_KEY_NAME, ciphertext=ciphertext_str, mount_point=self.mount_path- )+ async def derive_account_key(self, account_id: str) -> bytes:+ """+ Derive Account Key using HKDF.- import base64+ Args:+ account_id: Account ID- return base64.b64decode(response["data"]["plaintext"])+ Returns:+ Derived Account Key+ """+ root_key = await self.get_root_key()+ return await self._hkdf_derive(root_key, account_id, HKDF_SALT, HKDF_INFO_PREFIX)-class VolcengineKMSProvider(RootKeyProvider):+class VolcengineKMSProvider(BaseProvider):"""Volcengine KMS Key Provider.Suitable for production environments, using Volcengine KMS service for key management.Core features:- - Root key storage: Volcengine KMS+ - Root key storage: Volcengine KMS (encrypted)- Account Key derivation: HKDF-SHA256- - File Key encryption: Volcengine KMS encryption API+ - File Key encryption: AES-256-GCM (with Account Key)"""- SALT = b"OpenViking_KDF_Salt"- INFO_PREFIX = b"OpenViking_Account_"+ ROOT_KEY_FILENAME = "openviking-volcengine-root-key.enc"def __init__(self,@@ -365,6 +459,7 @@ class VolcengineKMSProvider(RootKeyProvider):secret_access_key: str,key_id: str,endpoint: Optional[str] = None,+ key_file: Optional[str] = None,):"""Initialize Volcengine KMS Provider.@@ -375,13 +470,19 @@ class VolcengineKMSProvider(RootKeyProvider):secret_access_key: Volcengine Secret Access Keykey_id: Volcengine KMS Key ID (immutable system-generated identifier)endpoint: Custom KMS service endpoint (optional)+ key_file: Path to encrypted root key file (default: ~/.openviking/openviking-volcengine-root-key.enc)"""self.region = regionself.access_key_id = access_key_idself.secret_access_key = secret_access_keyself.key_id = key_idself.endpoint = endpoint or f"kms.{region}.volcengineapi.com"+ if key_file:+ self.key_file = Path(key_file).expanduser()+ else:+ self.key_file = Path.home() / ".openviking" / self.ROOT_KEY_FILENAMEself._kms_client = None+ self._root_key: Optional[bytes] = Noneasync def _get_kms_client(self):"""@@ -433,11 +534,13 @@ class VolcengineKMSProvider(RootKeyProvider):}return api_info- def encrypt(self, key_id, plaintext):+ def encrypt(self, key_id, plaintext, encryption_context=None):body = {"KeyID": key_id,"Plaintext": base64.b64encode(plaintext).decode("utf-8"),}+ if encryption_context:+ body["EncryptionContext"] = encryption_contextres = self.json("Encrypt", {}, json.dumps(body))if res == "":@@ -449,11 +552,13 @@ class VolcengineKMSProvider(RootKeyProvider):return base64.b64decode(res_json["Result"]["CiphertextBlob"])raise Exception(f"Unexpected response: {res_json}")- def decrypt(self, ciphertext_blob, key_id):+ def decrypt(self, ciphertext_blob, key_id, encryption_context=None):body = {"KeyID": key_id,"CiphertextBlob": base64.b64encode(ciphertext_blob).decode("utf-8"),}+ if encryption_context:+ body["EncryptionContext"] = encryption_contextres = self.json("Decrypt", {}, json.dumps(body))if res == "":@@ -470,83 +575,96 @@ class VolcengineKMSProvider(RootKeyProvider):)return self._kms_client- async def get_root_key(self) -> bytes:+ async def _encrypt_with_kms(self, plaintext: bytes) -> bytes:"""- Get root key (only used for deriving Account Key).+ Encrypt data with Volcengine KMS.- Note: In Volcengine KMS, the root key is managed by KMS service.- Here we return the base key used for deriving Account Key.+ Args:+ plaintext: Plaintext dataReturns:- Base key+ Encrypted data"""- return b"OpenViking_Volcengine_KMS_Root_Seed"+ client = await self._get_kms_client()+ return await asyncio.to_thread(client.encrypt, self.key_id, plaintext)- async def derive_account_key(self, account_id: str) -> bytes:+ async def _decrypt_with_kms(self, ciphertext: bytes) -> bytes:"""- Derive Account Key using HKDF.+ Decrypt data with Volcengine KMS.Args:- account_id: Account ID+ ciphertext: Encrypted dataReturns:- Derived Account Key+ Decrypted data"""- root_key = await self.get_root_key()- return await self._hkdf_derive(root_key, account_id)+ client = await self._get_kms_client()+ return await asyncio.to_thread(client.decrypt, ciphertext, self.key_id)- async def _hkdf_derive(self, root_key: bytes, account_id: str) -> bytes:+ async def _get_or_create_root_key(self) -> bytes:"""- Derive key using HKDF.-- Args:- root_key: Root key- account_id: Account ID+ Get or create root key.Returns:- Derived key+ Root key bytes"""+ if self._root_key is not None:+ return self._root_key+try:- from cryptography.hazmat.primitives import hashes- from cryptography.hazmat.primitives.kdf.hkdf import HKDF+ # Try to read encrypted root key from file+ if self.key_file.exists():+ with open(self.key_file, "rb") as f:+ encrypted_root_key = f.read()+ self._root_key = await self._decrypt_with_kms(encrypted_root_key)+ logger.info(f"Loaded existing root key from {self.key_file}")+ return self._root_key+ except Exception as e:+ raise ConfigError(+ f"Failed to load existing root key from {self.key_file}. Cannot continue: {e}"+ )- hkdf = HKDF(- algorithm=hashes.SHA256(),- length=32,- salt=self.SALT,- info=self.INFO_PREFIX + account_id.encode(),+ # Generate new root key+ import secrets++ self._root_key = secrets.token_bytes(32)++ # Encrypt and store root key+ try:+ encrypted_root_key = await self._encrypt_with_kms(self._root_key)+ self.key_file.parent.mkdir(parents=True, exist_ok=True)+ with open(self.key_file, "wb") as f:+ f.write(encrypted_root_key)+ logger.info(f"Created and stored new root key at {self.key_file}")+ except Exception as e:+ raise ConfigError(+ f"Failed to persist root key to file. "+ f"Refusing to start with ephemeral key (data loss risk): {e}")- return hkdf.derive(root_key)- except ImportError:- raise ConfigError("cryptography library is required for encryption")- async def encrypt_file_key(self, plaintext_key: bytes, account_id: str) -> bytes:- """- Encrypt File Key using Volcengine KMS.+ return self._root_key- Args:- plaintext_key: Plaintext File Key- account_id: Account ID (not used in Volcengine KMS mode)+ async def get_root_key(self) -> bytes:+ """+ Get root key.Returns:- Encrypted File Key+ Root key"""- client = await self._get_kms_client()- return client.encrypt(self.key_id, plaintext_key)+ return await self._get_or_create_root_key()- async def decrypt_file_key(self, encrypted_key: bytes, account_id: str) -> bytes:+ async def derive_account_key(self, account_id: str) -> bytes:"""- Decrypt File Key using Volcengine KMS.+ Derive Account Key using HKDF.Args:- encrypted_key: Encrypted File Key- account_id: Account ID (not used in Volcengine KMS mode)+ account_id: Account IDReturns:- Decrypted File Key+ Derived Account Key"""- client = await self._get_kms_client()- return client.decrypt(encrypted_key, self.key_id)+ root_key = await self.get_root_key()+ return await self._hkdf_derive(root_key, account_id, HKDF_SALT, HKDF_INFO_PREFIX)def create_root_key_provider(@@ -576,10 +694,24 @@ def create_root_key_provider(address = vault_config.get("address")token = vault_config.get("token")mount_point = vault_config.get("mount_point", "transit")+ kv_mount_point = vault_config.get("kv_mount_point", "secret")+ kv_version = vault_config.get("kv_version", 1)+ root_key_name = vault_config.get("root_key_name", "openviking-root-key")+ encrypted_root_key_key = vault_config.get(+ "encrypted_root_key_key", "openviking-encrypted-root-key"+ )if not address or not token:raise ConfigError("vault.address and vault.token are required")- return VaultProvider(address, token, mount_point)+ return VaultProvider(+ address,+ token,+ mount_point,+ kv_mount_point,+ kv_version,+ root_key_name,+ encrypted_root_key_key,+ )elif provider_type == "volcengine_kms":volc_config = config.get("volcengine_kms", {})@@ -587,10 +719,14 @@ def create_root_key_provider(access_key = volc_config.get("access_key")secret_key = volc_config.get("secret_key")key_id = volc_config.get("key_id")+ endpoint = volc_config.get("endpoint")+ key_file = volc_config.get("key_file")if not all([region, access_key, secret_key, key_id]):raise ConfigError("volcengine_kms region, access_key, secret_key, key_id are required")- return VolcengineKMSProvider(region, access_key, secret_key, key_id)+ return VolcengineKMSProvider(+ region, access_key, secret_key, key_id, endpoint=endpoint, key_file=key_file+ )else:raise ConfigError(f"Unsupported provider type: {provider_type}")diff --git a/openviking/server/api_keys.py b/openviking/server/api_keys.pyindex 98b361d4a..636656de8 100644--- a/openviking/server/api_keys.py+++ b/openviking/server/api_keys.py@@ -498,11 +498,10 @@ class APIKeyManager:raw = content.content if hasattr(content, "content") else b""# Decrypt content if encryption is enabled- if self._viking_fs._encryptor:- # Extract account ID from path- parts = path.split("/")- account_id = parts[2] if len(parts) >= 3 else "default"- raw = await self._viking_fs._encryptor.decrypt(account_id, raw)+ # Extract account ID from path+ parts = path.split("/")+ account_id = parts[2] if len(parts) >= 3 else "default"+ raw = await self._viking_fs.decrypt_bytes(account_id, raw)text = raw.decode("utf-8") if isinstance(raw, bytes) else rawreturn json.loads(text)@@ -516,11 +515,10 @@ class APIKeyManager:content = content.encode("utf-8")# Encrypt content if encryption is enabled- if self._viking_fs._encryptor:- # Extract account ID from path- parts = path.split("/")- account_id = parts[2] if len(parts) >= 3 else "default"- content = await self._viking_fs._encryptor.encrypt(account_id, content)+ # Extract account ID from path+ parts = path.split("/")+ account_id = parts[2] if len(parts) >= 3 else "default"+ content = await self._viking_fs.encrypt_bytes(account_id, content)# Ensure parent directories existself._ensure_parent_dirs(path)diff --git a/openviking/storage/viking_fs.py b/openviking/storage/viking_fs.pyindex 100adaf71..744271dce 100644--- a/openviking/storage/viking_fs.py+++ b/openviking/storage/viking_fs.py@@ -204,6 +204,36 @@ class VikingFS:real_ctx = self._ctx_or_default(ctx)return await self._encryptor.decrypt(real_ctx.account_id, content)+ async def encrypt_bytes(self, account_id: str, data: bytes) -> bytes:+ """+ Encrypt bytes using the encryptor for the specified account.++ Args:+ account_id: Account ID to use for encryption+ data: Bytes to encrypt++ Returns:+ Encrypted bytes, or original bytes if encryption is disabled+ """+ if not self._encryptor:+ return data+ return await self._encryptor.encrypt(account_id, data)++ async def decrypt_bytes(self, account_id: str, data: bytes) -> bytes:+ """+ Decrypt bytes using the encryptor for the specified account.++ Args:+ account_id: Account ID to use for decryption+ data: Bytes to decrypt++ Returns:+ Decrypted bytes, or original bytes if encryption is disabled+ """+ if not self._encryptor:+ return data+ return await self._encryptor.decrypt(account_id, data)+@contextmanagerdef bind_request_context(self, ctx: RequestContext):"""Temporarily bind ctx for legacy internal call paths without explicit ctx param."""@@ -258,18 +288,35 @@ class VikingFS:"""Read file"""self._ensure_access(uri, ctx)path = self._uri_to_path(uri, ctx=ctx)- result = self.agfs.read(path, offset, size)- if isinstance(result, bytes):- raw = result- elif result is not None and hasattr(result, "content"):- raw = result.content- else:- raw = b""- # If encryption is enabled and not reading from offset, try to decrypt- if self._encryptor and offset == 0 and size == -1:+ if self._encryptor:+ # When encryption is enabled: must read entire file for decryption+ result = self.agfs.read(path, 0, -1)+ if isinstance(result, bytes):+ raw = result+ elif result is not None and hasattr(result, "content"):+ raw = result.content+ else:+ raw = b""+raw = await self._decrypt_content(raw, ctx=ctx)+ # Apply slicing on decrypted plaintext+ if offset > 0 or size != -1:+ if size != -1:+ raw = raw[offset : offset + size]+ else:+ raw = raw[offset:]+ else:+ # When not encrypted: normal read+ result = self.agfs.read(path, offset, size)+ if isinstance(result, bytes):+ raw = result+ elif result is not None and hasattr(result, "content"):+ raw = result.content+ else:+ raw = b""+return rawasync def write(@@ -1558,8 +1605,8 @@ class VikingFS:else:raw = b""- # If encryption is enabled and not reading from offset, try to decrypt- if self._encryptor and offset == 0 and limit == -1:+ # If encryption is enabled, always decrypt full file first+ if self._encryptor:raw = await self._decrypt_content(raw, ctx=ctx)text = self._decode_bytes(raw)@@ -1684,13 +1731,13 @@ class VikingFS:if len(all_entries) >= node_limit:breakname = entry.get("name", "")- # 修改后:通过截断字符串来兼容 7 位或更多位的微秒+ # After modification: compatible with 7+ digits of microseconds by truncatingraw_time = entry.get("modTime", "")if raw_time and len(raw_time) > 26 and "+" in raw_time:- # 处理像 2026-02-21T13:20:23.1470042+08:00 这样的字符串- # 截断为 2026-02-21T13:20:23.147004+08:00+ # Handle strings like 2026-02-21T13:20:23.1470042+08:00+ # Truncate to 2026-02-21T13:20:23.147004+08:00parts = raw_time.split("+")- # 保持时间部分最多 26 位 (YYYY-MM-DDTHH:MM:SS.mmmmmm)+ # Keep time part at most 26 characters (YYYY-MM-DDTHH:MM:SS.mmmmmm)raw_time = parts[0][:26] + "+" + parts[1]new_entry = {"uri": self._path_to_uri(f"{path}/{name}", ctx=ctx),diff --git a/tests/integration/conftest.py b/tests/integration/conftest.pyindex 4ca8d9d9f..6ca667509 100644--- a/tests/integration/conftest.py+++ b/tests/integration/conftest.py@@ -31,6 +31,21 @@ TEST_TMP_DIR = PROJECT_ROOT / "test_data" / "tmp_integration"GOOGLE_API_KEY = os.environ.get("GOOGLE_API_KEY", "")requires_api_key = pytest.mark.skipif(not GOOGLE_API_KEY, reason="GOOGLE_API_KEY not set")+# ── Vault integration test helpers ──────────────────────────────────────────+VAULT_ADDR = os.environ.get("VAULT_ADDR", "http://127.0.0.1:8200")+VAULT_TOKEN = os.environ.get("VAULT_TOKEN", "")+requires_vault = pytest.mark.skipif(not VAULT_TOKEN, reason="VAULT_TOKEN not set")++# ── Volcengine KMS integration test helpers ──────────────────────────────────────────+VOLCENGINE_ACCESS_KEY = os.environ.get("VOLCENGINE_ACCESS_KEY", "")+VOLCENGINE_SECRET_KEY = os.environ.get("VOLCENGINE_SECRET_KEY", "")+VOLCENGINE_KMS_KEY_ID = os.environ.get("VOLCENGINE_KMS_KEY_ID", "")+VOLCENGINE_KMS_REGION = os.environ.get("VOLCENGINE_KMS_REGION", "cn-beijing")+requires_volcengine_kms = pytest.mark.skipif(+ not (VOLCENGINE_ACCESS_KEY and VOLCENGINE_SECRET_KEY and VOLCENGINE_KMS_KEY_ID),+ reason="VOLCENGINE_ACCESS_KEY, VOLCENGINE_SECRET_KEY, or VOLCENGINE_KMS_KEY_ID not set",+)+# (model_name, default_dimension, token_limit)GEMINI_MODELS = [("gemini-embedding-2-preview", 3072, 8192),diff --git a/tests/integration/test_encryption_integration.py b/tests/integration/test_encryption_integration.pyindex e5486c0b9..ecbdd5d47 100644--- a/tests/integration/test_encryption_integration.py+++ b/tests/integration/test_encryption_integration.py@@ -1092,6 +1092,110 @@ This is a test skill for verifying encryption functionality.print("✅ All tests completed!")print("=" * 80)+ async def test_read_file_with_offset_and_limit_encryption(+ self, openviking_service_with_encryption+ ):+ """+ Test read_file() with offset and limit returns correct plaintext when encryption is enabled.+ Verifies that partial reads (by line) work correctly with encryption.+ """+ data = openviking_service_with_encryption+ svc = data["service"]++ # Create request context+ from openviking.server.identity import RequestContext, Role+ from openviking_cli.session.user_id import UserIdentifier++ default_user = UserIdentifier("default", "default", "default")+ ctx = RequestContext(user=default_user, role=Role.ROOT)++ # Write a multi-line test file+ test_lines = [+ "Line 0: This is the first line",+ "Line 1: Second line content",+ "Line 2: Third line here",+ "Line 3: Fourth line",+ "Line 4: Fifth and final line",+ ]+ test_content = "\n".join(test_lines)+ test_uri = "viking://default/test_multiline.txt"++ await svc.viking_fs.write_file(test_uri, test_content, ctx=ctx)++ # Test 1: Read with offset=0, limit=-1 (full file)+ full_content = await svc.viking_fs.read_file(test_uri, offset=0, limit=-1, ctx=ctx)+ assert full_content == test_content, "Full file read should return correct content"++ # Test 2: Read with offset=1, limit=3 (lines 1, 2, 3)+ partial_content = await svc.viking_fs.read_file(test_uri, offset=1, limit=3, ctx=ctx)+ expected_lines = test_lines[1:4]+ expected_content = "\n".join(expected_lines)+ # read_file() 会在最后一行后添加换行符,所以需要处理这种情况+ assert partial_content.rstrip("\n") == expected_content, "Partial read failed"++ # Test 3: Read with offset=3, limit=-1 (from line 3 to end)+ from_line_3 = await svc.viking_fs.read_file(test_uri, offset=3, limit=-1, ctx=ctx)+ expected_from_line_3 = "\n".join(test_lines[3:])+ assert from_line_3.rstrip("\n") == expected_from_line_3, "Read from offset failed"++ # Verify file is encrypted on disk+ agfs_client = svc._agfs_client+ agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)+ raw_content = agfs_client.read(agfs_path)+ assert raw_content.startswith(b"OVE1"), "File should be encrypted"++ print("\n" + "=" * 80)+ print("✅ read_file() with offset/limit encryption test completed!")+ print("=" * 80)++ async def test_read_with_offset_and_size_encryption(self, openviking_service_with_encryption):+ """+ Test read() with offset and size returns correct plaintext when encryption is enabled.+ Verifies that partial reads (by byte) work correctly with encryption.+ """+ data = openviking_service_with_encryption+ svc = data["service"]++ # Create request context+ from openviking.server.identity import RequestContext, Role+ from openviking_cli.session.user_id import UserIdentifier++ default_user = UserIdentifier("default", "default", "default")+ ctx = RequestContext(user=default_user, role=Role.ROOT)++ # Write a test file+ test_content = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"+ test_uri = "viking://default/test_bytes.txt"++ await svc.viking_fs.write_file(test_uri, test_content.decode("utf-8"), ctx=ctx)++ # Test 1: Read with offset=0, size=-1 (full file)+ full_bytes = await svc.viking_fs.read(test_uri, offset=0, size=-1, ctx=ctx)+ # read() 返回的字节会在末尾多一个换行符,所以使用 rstrip(b"\n")+ assert full_bytes.rstrip(b"\n") == test_content, (+ "Full file read should return correct bytes"+ )++ # Test 2: Read with offset=5, size=10 (bytes 5-14)+ partial_bytes = await svc.viking_fs.read(test_uri, offset=5, size=10, ctx=ctx)+ expected_bytes = test_content[5:15]+ assert partial_bytes == expected_bytes, "Partial read failed"++ # Test 3: Read with offset=10, size=-1 (from byte 10 to end)+ from_byte_10 = await svc.viking_fs.read(test_uri, offset=10, size=-1, ctx=ctx)+ expected_from_byte_10 = test_content[10:]+ assert from_byte_10.rstrip(b"\n") == expected_from_byte_10, "Read from offset failed"++ # Verify file is encrypted on disk+ agfs_client = svc._agfs_client+ agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)+ raw_content = agfs_client.read(agfs_path)+ assert raw_content.startswith(b"OVE1"), "File should be encrypted"++ print("\n" + "=" * 80)+ print("✅ read() with offset/size encryption test completed!")+ print("=" * 80)+class TestAddResourceWithSemanticProcessing:"""diff --git a/tests/integration/test_vault_encryption_integration.py b/tests/integration/test_vault_encryption_integration.pynew file mode 100644index 000000000..25925306a--- /dev/null+++ b/tests/integration/test_vault_encryption_integration.py@@ -0,0 +1,716 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0++"""+Vault encryption integration tests.++Tests encryption functionality with HashiCorp Vault as the key provider.+Requires VAULT_TOKEN env var to be set.+Run: pytest tests/integration/test_vault_encryption_integration.py -v -m integration+"""++import secrets+from pathlib import Path++import pytest+import pytest_asyncio++from openviking import AsyncOpenViking+from openviking.crypto.config import bootstrap_encryption+from openviking.crypto.encryptor import FileEncryptor+from openviking.crypto.exceptions import AuthenticationFailedError, ConfigError+from openviking.crypto.providers import VaultProvider, create_root_key_provider+from openviking.server.api_keys import APIKeyManager+from openviking.service.core import OpenVikingService+from openviking_cli.session.user_id import UserIdentifier+from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton+from tests.integration.conftest import VAULT_ADDR, VAULT_TOKEN, requires_vault++pytestmark = [pytest.mark.integration, requires_vault]++# Vault environment configuration+VAULT_MOUNT_PATH = "transit"+++class TestVaultProviderIntegration:+ """Integration tests for VaultProvider."""++ @pytest_asyncio.fixture(scope="function")+ async def vault_provider(self):+ """Fixture that provides a VaultProvider instance."""+ provider = VaultProvider(addr=VAULT_ADDR, token=VAULT_TOKEN, mount_path=VAULT_MOUNT_PATH)+ yield provider++ @pytest.fixture(scope="function")+ def vault_config(self):+ """Fixture that provides Vault configuration."""+ return {+ "vault": {+ "address": VAULT_ADDR,+ "token": VAULT_TOKEN,+ "mount_point": VAULT_MOUNT_PATH,+ }+ }++ async def test_vault_provider_initialization(self, vault_provider):+ """Test VaultProvider initialization and Vault connection."""+ client = await vault_provider._get_client()+ assert client is not None+ assert client.is_authenticated()++ async def test_derive_account_key(self, vault_provider):+ """Test deriving account keys."""+ account_id = "test-account-vault"+ account_key = await vault_provider.derive_account_key(account_id)++ assert account_key is not None+ assert isinstance(account_key, bytes)+ assert len(account_key) == 32++ async def test_derive_different_account_keys(self, vault_provider):+ """Test that different accounts get different keys."""+ account1_key = await vault_provider.derive_account_key("account-1")+ account2_key = await vault_provider.derive_account_key("account-2")++ assert account1_key != account2_key++ async def test_derive_same_account_key_consistent(self, vault_provider):+ """Test that same account gets consistent key."""+ key1 = await vault_provider.derive_account_key("test-account")+ key2 = await vault_provider.derive_account_key("test-account")++ assert key1 == key2++ async def test_encrypt_decrypt_file_key(self, vault_provider):+ """Test encrypting and decrypting file keys."""+ plaintext_key = secrets.token_bytes(32)+ account_id = "test-account-encrypt"++ encrypted_key, iv = await vault_provider.encrypt_file_key(plaintext_key, account_id)++ assert encrypted_key is not None+ assert isinstance(encrypted_key, bytes)+ assert encrypted_key != plaintext_key+ assert iv is not None+ assert isinstance(iv, bytes)++ decrypted = await vault_provider.decrypt_file_key(encrypted_key, iv, account_id)++ assert decrypted == plaintext_key++ async def test_encrypt_decrypt_multiple_keys(self, vault_provider):+ """Test encrypting and decrypting multiple different keys."""+ key1 = secrets.token_bytes(32)+ key2 = secrets.token_bytes(32)+ account_id = "test-account-multiple"++ encrypted1, iv1 = await vault_provider.encrypt_file_key(key1, account_id)+ encrypted2, iv2 = await vault_provider.encrypt_file_key(key2, account_id)++ assert encrypted1 != encrypted2++ decrypted1 = await vault_provider.decrypt_file_key(encrypted1, iv1, account_id)+ decrypted2 = await vault_provider.decrypt_file_key(encrypted2, iv2, account_id)++ assert decrypted1 == key1+ assert decrypted2 == key2++ async def test_create_root_key_provider_vault(self, vault_config):+ """Test creating VaultProvider via create_root_key_provider."""+ provider = create_root_key_provider("vault", vault_config)++ assert provider is not None+ assert isinstance(provider, VaultProvider)++ async def test_get_root_key(self, vault_provider):+ """Test getting root key."""+ root_key = await vault_provider.get_root_key()++ assert root_key is not None+ assert isinstance(root_key, bytes)+ assert len(root_key) > 0++ async def test_full_roundtrip(self, vault_provider):+ """Test full encryption/decryption roundtrip."""+ # Derive account key+ account_id = "test-account-roundtrip"+ account_key = await vault_provider.derive_account_key(account_id)++ assert account_key is not None+ assert len(account_key) == 32++ # Generate and encrypt file key+ file_key = secrets.token_bytes(32)+ encrypted_key, iv = await vault_provider.encrypt_file_key(file_key, account_id)++ assert encrypted_key is not None+ assert isinstance(encrypted_key, bytes)+ assert iv is not None+ assert isinstance(iv, bytes)++ # Decrypt and verify+ decrypted_key = await vault_provider.decrypt_file_key(encrypted_key, iv, account_id)++ assert decrypted_key == file_key+++class TestVaultProviderErrors:+ """Test error handling for VaultProvider."""++ async def test_invalid_token(self):+ """Test with invalid token."""+ provider = VaultProvider(+ addr="http://127.0.0.1:8200", token="invalid-token", mount_path="transit"+ )++ with pytest.raises(AuthenticationFailedError):+ await provider._get_client()++ async def test_invalid_address(self):+ """Test with invalid address."""+ provider = VaultProvider(+ addr="http://invalid-address-that-does-not-exist:8200",+ token="test-token",+ mount_path="transit",+ )++ with pytest.raises(Exception): # noqa: B017+ await provider._get_client()++ def test_create_provider_missing_config(self):+ """Test creating provider with missing config."""+ with pytest.raises(ConfigError):+ create_root_key_provider("vault", {})++ def test_create_provider_missing_token(self):+ """Test creating provider with missing token."""+ with pytest.raises(ConfigError):+ create_root_key_provider("vault", {"vault": {"address": "http://127.0.0.1:8200"}})++ def test_create_provider_missing_address(self):+ """Test creating provider with missing address."""+ with pytest.raises(ConfigError):+ create_root_key_provider("vault", {"vault": {"token": "test-token"}})+++@pytest_asyncio.fixture(scope="function")+async def vault_encryption_config():+ """Fixture that provides encryption configuration with Vault provider"""+ return {+ "encryption": {+ "enabled": True,+ "provider": "vault",+ "vault": {"address": VAULT_ADDR, "token": VAULT_TOKEN, "mount_point": VAULT_MOUNT_PATH},+ }+ }+++@pytest_asyncio.fixture(scope="function")+async def vault_file_encryptor():+ """Fixture that provides a FileEncryptor instance with Vault provider for testing"""+ provider = VaultProvider(addr=VAULT_ADDR, token=VAULT_TOKEN, mount_path=VAULT_MOUNT_PATH)+ return FileEncryptor(provider)+++@pytest_asyncio.fixture(scope="function")+async def openviking_client_with_vault_encryption(test_data_dir: Path, vault_encryption_config):+ """Fixture that provides an OpenViking client with Vault encryption enabled"""+ await AsyncOpenViking.reset()+ OpenVikingConfigSingleton.reset_instance()++ # Clean data directory+ if test_data_dir.exists():+ import shutil++ shutil.rmtree(test_data_dir)+ test_data_dir.mkdir(parents=True, exist_ok=True)++ # Create config dict with encryption enabled+ config_dict = {}+ config_dict.update(vault_encryption_config)+ config_dict["storage"] = {+ "workspace": str(test_data_dir / "workspace"),+ "vectordb": {"name": "test", "backend": "local", "project": "default"},+ }+ config_dict["embedding"] = {+ "dense": {+ "model": "doubao-embedding-vision-250615",+ "api_key": "2a2284d9-79a3-4978-b718-dfeffdda04ad",+ "api_base": "https://ark.cn-beijing.volces.com/api/v3",+ "dimension": 1024,+ "provider": "volcengine",+ }+ }++ # Initialize config singleton+ OpenVikingConfigSingleton.initialize(config_dict=config_dict)++ client = AsyncOpenViking(path=str(test_data_dir))+ await client.initialize()++ yield client++ await client.close()+ await AsyncOpenViking.reset()+ OpenVikingConfigSingleton.reset_instance()+++class TestVaultEncryptionBootstrap:+ """Tests for encryption module bootstrap with Vault provider"""++ async def test_bootstrap_encryption_vault_enabled(self, vault_encryption_config):+ """Test bootstrapping encryption with Vault provider enabled"""+ encryptor = await bootstrap_encryption(vault_encryption_config)+ assert encryptor is not None+ assert isinstance(encryptor, FileEncryptor)+ assert isinstance(encryptor.provider, VaultProvider)+++class TestVaultFileEncryptorIntegration:+ """Tests for FileEncryptor integration with Vault provider"""++ async def test_encrypt_decrypt_roundtrip_vault(self, vault_file_encryptor):+ """Test basic encryption/decryption roundtrip with Vault"""+ account_id = "test_account_vault"+ plaintext = b"This is a test file content for Vault encryption integration"++ # Encrypt+ ciphertext = await vault_file_encryptor.encrypt(account_id, plaintext)+ assert ciphertext != plaintext+ assert ciphertext.startswith(b"OVE1")++ # Decrypt+ decrypted = await vault_file_encryptor.decrypt(account_id, ciphertext)+ assert decrypted == plaintext++ async def test_different_accounts_isolation(self, vault_file_encryptor):+ """Test that different accounts cannot decrypt each other's data"""+ account1 = "account_1_vault"+ account2 = "account_2_vault"+ plaintext = b"Sensitive data for account isolation test"++ # Encrypt with account1+ ciphertext = await vault_file_encryptor.encrypt(account1, plaintext)++ # Decrypt with account1 should work+ decrypted1 = await vault_file_encryptor.decrypt(account1, ciphertext)+ assert decrypted1 == plaintext++ # Decrypt with account2 should fail+ from openviking.crypto.exceptions import KeyMismatchError++ with pytest.raises(KeyMismatchError):+ await vault_file_encryptor.decrypt(account2, ciphertext)++ async def test_encrypt_empty_data_vault(self, vault_file_encryptor):+ """Test encrypting and decrypting empty data with Vault"""+ account_id = "test_account_vault"+ plaintext = b""++ ciphertext = await vault_file_encryptor.encrypt(account_id, plaintext)+ assert ciphertext.startswith(b"OVE1")++ decrypted = await vault_file_encryptor.decrypt(account_id, ciphertext)+ assert decrypted == b""+++class TestVikingFSEncryptionWithVault:+ """+ Complete VikingFS encryption tests using Vault as key provider.+ Includes account management, resource operations, skill operations, etc.+ """++ ROOT_KEY = "test-root-key-for-vault-encryption-tests-abcdef123456"++ @pytest_asyncio.fixture(scope="function")+ async def openviking_service_with_vault_encryption(+ self, test_data_dir: Path, vault_encryption_config+ ):+ """+ Fixture provides OpenVikingService instance with Vault encryption.+ Also initializes APIKeyManager for account management.+ """+ # Clean data directory+ if test_data_dir.exists():+ import shutil++ shutil.rmtree(test_data_dir)+ test_data_dir.mkdir(parents=True, exist_ok=True)++ # Create config dict with encryption enabled and root_api_key set+ config_dict = {}+ config_dict.update(vault_encryption_config)+ config_dict["storage"] = {+ "workspace": str(test_data_dir / "workspace"),+ "vectordb": {"name": "test", "backend": "local", "project": "default"},+ }+ config_dict["embedding"] = {+ "dense": {+ "model": "doubao-embedding-vision-250615",+ "api_key": "2a2284d9-79a3-4978-b718-dfeffdda04ad",+ "api_base": "https://ark.cn-beijing.volces.com/api/v3",+ "dimension": 1024,+ "provider": "volcengine",+ }+ }+ config_dict["server"] = {"root_api_key": self.ROOT_KEY}++ # Initialize config singleton+ OpenVikingConfigSingleton.initialize(config_dict=config_dict)++ # Create OpenVikingService+ svc = OpenVikingService(+ path=str(test_data_dir / "viking"), user=UserIdentifier.the_default_user("test_user")+ )+ await svc.initialize()++ # Create APIKeyManager using VikingFS to ensure system files are encrypted+ api_key_manager = APIKeyManager(+ root_key=self.ROOT_KEY, viking_fs=svc.viking_fs, encryption_enabled=True+ )+ await api_key_manager.load()++ yield {"service": svc, "api_key_manager": api_key_manager, "test_data_dir": test_data_dir}++ await svc.close()+ await AsyncOpenViking.reset()+ OpenVikingConfigSingleton.reset_instance()++ def _is_file_encrypted(self, file_path: Path) -> bool:+ """+ Check if file is encrypted (by "OVE1" header).++ Args:+ file_path: File path++ Returns:+ True if file is encrypted, False otherwise+ """+ if not file_path.exists():+ return False+ try:+ with open(file_path, "rb") as f:+ header = f.read(4)+ return header == b"OVE1"+ except Exception:+ return False++ async def _check_all_files_encrypted(+ self, svc, ctx, base_uri: str, print_paths: bool = True+ ) -> None:+ """+ Recursively check if all files in directory are encrypted.++ Args:+ svc: OpenVikingService instance+ ctx: RequestContext instance+ base_uri: Base URI to check+ print_paths: Whether to print encrypted file paths+ """+ agfs_client = svc._agfs_client++ async def _check_recursive(uri: str):+ try:+ entries = await svc.viking_fs.ls(uri, ctx=ctx)+ for entry in entries:+ entry_uri = entry["uri"]+ if entry["isDir"]:+ await _check_recursive(entry_uri)+ else:+ # Use VikingFS's _uri_to_path method to get file path+ agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)+ raw_content = agfs_client.read(agfs_path)+ assert raw_content.startswith(b"OVE1"), f"File {entry_uri} is not encrypted"+ if print_paths:+ print(f"✓ File is encrypted: {entry_uri}")+ except Exception as e:+ if "not found" in str(e).lower():+ pass+ else:+ raise++ await _check_recursive(base_uri)++ @pytest.mark.asyncio+ async def test_resource_operations_with_encryption(+ self, openviking_service_with_vault_encryption, tmp_path+ ):+ """Test 4: Resource operations and encryption verification"""+ data = openviking_service_with_vault_encryption+ svc = data["service"]+ api_key_manager = data["api_key_manager"]++ # Create test account+ account_id = "test-account-resources"+ admin_user_id = "admin"+ await api_key_manager.create_account(account_id, admin_user_id)++ from openviking.server.identity import RequestContext, Role++ user = UserIdentifier(account_id, admin_user_id, admin_user_id)+ ctx = RequestContext(user=user, role=Role.ADMIN)++ # Create test resource file+ test_content = "This is test resource content for Vault encryption"+ test_uri = f"viking://{account_id}/resources/test_file.txt"++ # Write file+ await svc.viking_fs.write_file(test_uri, test_content, ctx=ctx)++ # Verify read content is correct+ read_content = await svc.viking_fs.read_file(test_uri, ctx=ctx)+ assert read_content == test_content++ # Verify file is encrypted+ agfs_client = svc._agfs_client+ agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)+ raw_content = agfs_client.read(agfs_path)+ assert raw_content.startswith(b"OVE1")++ # Test various operations+ resources_dir_uri = f"viking://{account_id}/resources"++ # ls operation+ ls_entries = await svc.viking_fs.ls(resources_dir_uri, ctx=ctx)+ assert len(ls_entries) > 0++ # tree operation+ tree_entries = await svc.viking_fs.tree(resources_dir_uri, ctx=ctx)+ assert len(tree_entries) > 0++ # grep operation+ try:+ grep_result = await svc.viking_fs.grep(resources_dir_uri, "Vault", ctx=ctx)+ assert grep_result is not None+ except Exception as e:+ print(f"[WARNING] grep operation may not be supported: {e}")++ # abstract operation+ try:+ abstract = await svc.viking_fs.abstract(test_uri, ctx=ctx)+ assert abstract is not None+ assert "OVE1" not in abstract+ except Exception as e:+ print(f"[WARNING] abstract operation may not be supported: {e}")++ # overview operation+ try:+ overview = await svc.viking_fs.overview(test_uri, ctx=ctx)+ assert overview is not None+ assert "OVE1" not in overview+ except Exception as e:+ print(f"[WARNING] overview operation may not be supported: {e}")++ @pytest.mark.asyncio+ async def test_skill_operations_with_encryption(self, openviking_service_with_vault_encryption):+ """Test 5: Skill operations and encryption verification"""+ data = openviking_service_with_vault_encryption+ svc = data["service"]+ api_key_manager = data["api_key_manager"]++ # Create test account+ account_id = "test-account-skills"+ admin_user_id = "admin"+ await api_key_manager.create_account(account_id, admin_user_id)++ from openviking.server.identity import RequestContext, Role++ user = UserIdentifier(account_id, admin_user_id, admin_user_id)+ ctx = RequestContext(user=user, role=Role.ADMIN)++ # Create skill directory and file+ skill_dir_uri = f"viking://{account_id}/agent/test-skill"+ skill_md_uri = f"{skill_dir_uri}/SKILL.md"++ # Create directory+ await svc.viking_fs.mkdir(skill_dir_uri, ctx=ctx)++ # Write skill file+ skill_content = """---+name: Test Skill+version: 1.0.0+description: Test skill for Vault encryption+---++# Test Skill++This is a test skill for verifying Vault encryption.+"""+ await svc.viking_fs.write_file(skill_md_uri, skill_content, ctx=ctx)++ # Verify read content is correct+ read_content = await svc.viking_fs.read_file(skill_md_uri, ctx=ctx)+ assert "Test Skill" in read_content++ # Verify file is encrypted+ agfs_client = svc._agfs_client+ agfs_path = svc.viking_fs._uri_to_path(skill_md_uri, ctx=ctx)+ raw_content = agfs_client.read(agfs_path)+ assert raw_content.startswith(b"OVE1")++ # Test various operations+ agent_dir_uri = f"viking://{account_id}/agent"++ # ls operation+ ls_entries = await svc.viking_fs.ls(agent_dir_uri, ctx=ctx)+ assert len(ls_entries) > 0++ # tree operation+ try:+ tree_entries = await svc.viking_fs.tree(agent_dir_uri, ctx=ctx)+ assert len(tree_entries) > 0+ except Exception as e:+ print(f"[WARNING] tree operation may not be supported: {e}")++ @pytest.mark.asyncio+ async def test_memory_operations_with_encryption(+ self, openviking_service_with_vault_encryption+ ):+ """Test 6: Memory operations and encryption verification"""+ data = openviking_service_with_vault_encryption+ svc = data["service"]+ api_key_manager = data["api_key_manager"]++ # Create test account+ account_id = "test-account-memories"+ user_id = "test-user"+ await api_key_manager.create_account(account_id, user_id)++ from openviking.server.identity import RequestContext, Role++ user = UserIdentifier(account_id, user_id, user_id)+ ctx = RequestContext(user=user, role=Role.USER)++ # Create memory directory and file+ memory_dir_uri = f"viking://{account_id}/user/{user_id}/memories"+ memory_file_uri = f"{memory_dir_uri}/preferences.md"++ # Create directory+ try:+ await svc.viking_fs.mkdir(memory_dir_uri, ctx=ctx)+ except Exception:+ pass # Directory may already exist++ # Write memory file+ memory_content = "# User Preferences\n\nTheme: dark\nLanguage: English"+ await svc.viking_fs.write_file(memory_file_uri, memory_content, ctx=ctx)++ # Verify read content is correct+ read_content = await svc.viking_fs.read_file(memory_file_uri, ctx=ctx)+ assert "User Preferences" in read_content++ # Verify file is encrypted+ agfs_client = svc._agfs_client+ agfs_path = svc.viking_fs._uri_to_path(memory_file_uri, ctx=ctx)+ raw_content = agfs_client.read(agfs_path)+ assert raw_content.startswith(b"OVE1")++ @pytest.mark.asyncio+ async def test_session_operations_with_encryption(+ self, openviking_service_with_vault_encryption+ ):+ """Test 7: Session operations and encryption verification"""+ data = openviking_service_with_vault_encryption+ svc = data["service"]+ api_key_manager = data["api_key_manager"]++ # Create test account+ account_id = "test-account-sessions"+ user_id = "test-user"+ await api_key_manager.create_account(account_id, user_id)++ from openviking.server.identity import RequestContext, Role++ user = UserIdentifier(account_id, user_id, user_id)+ ctx = RequestContext(user=user, role=Role.USER)++ # Create session+ session = await svc.sessions.create(ctx)+ session_id = session.session_id+ assert session_id is not None++ # Check if session directory files are encrypted+ session_dir_uri = f"viking://{account_id}/session"+ await self._check_all_files_encrypted(svc, ctx, session_dir_uri, print_paths=False)++ @pytest.mark.asyncio+ async def test_complete_encryption_workflow_with_vault(+ self, openviking_service_with_vault_encryption+ ):+ """+ Complete encryption workflow test using Vault as key provider, including:+ 1. Precondition: Create random account and user+ 2. Execution: Resource, skill, session, relationship operations+ 3. Encryption verification: Check OVE1 header through file system+ 4. Postcondition: Delete account+ """+ import uuid++ from openviking.server.identity import RequestContext, Role++ svc = openviking_service_with_vault_encryption["service"]+ api_key_manager = openviking_service_with_vault_encryption["api_key_manager"]+ openviking_service_with_vault_encryption["test_data_dir"]++ # 1. Precondition: Create random account and user+ random_account_id = f"test-account-{uuid.uuid4()}"+ random_user_id = f"test-user-{uuid.uuid4()}"+ print(f"\n=== Creating account: {random_account_id}, user: {random_user_id} ===")++ await api_key_manager.create_account(random_account_id, random_user_id)+ user = UserIdentifier(random_account_id, random_user_id, random_user_id)+ ctx = RequestContext(user=user, role=Role.ADMIN)++ # 2. Execution: Resource, skill, session operations+ print("\n=== Executing operations ===")++ # Create resource+ test_content = f"Test content for account {random_account_id} with Vault encryption"+ test_resource_uri = f"viking://{random_account_id}/resources/test_workflow.txt"+ await svc.viking_fs.write_file(test_resource_uri, test_content, ctx=ctx)+ print(f"✓ Resource created successfully: {test_resource_uri}")++ # Create skill+ skill_dir_uri = f"viking://{random_account_id}/agent/test-workflow-skill"+ skill_md_uri = f"{skill_dir_uri}/SKILL.md"+ await svc.viking_fs.mkdir(skill_dir_uri, ctx=ctx)+ skill_content = """---+name: Workflow Test Skill+version: 1.0.0+description: Test skill for complete workflow+---++# Workflow Test Skill+"""+ await svc.viking_fs.write_file(skill_md_uri, skill_content, ctx=ctx)+ print(f"✓ Skill created successfully: {skill_md_uri}")++ # Create memory+ memory_dir_uri = f"viking://{random_account_id}/user/{random_user_id}/memories"+ memory_file_uri = f"{memory_dir_uri}/test_memory.md"+ try:+ await svc.viking_fs.mkdir(memory_dir_uri, ctx=ctx)+ except Exception:+ pass+ memory_content = "# Test Memory\n\nThis is a test memory for workflow"+ await svc.viking_fs.write_file(memory_file_uri, memory_content, ctx=ctx)+ print(f"✓ Memory created successfully: {memory_file_uri}")++ # Create session+ session = await svc.sessions.create(ctx)+ session_id = session.session_id+ print(f"✓ Session created successfully: {session_id}")++ # 3. Encryption verification: Check OVE1 header through file system+ print("\n=== Encryption verification ===")++ # Check if all files are encrypted+ account_root_uri = f"viking://{random_account_id}"+ await self._check_all_files_encrypted(svc, ctx, account_root_uri, print_paths=True)++ # 4. Postcondition: Delete account+ print("\n=== Cleanup ===")+ await api_key_manager.delete_account(random_account_id)+ print(f"✓ Account {random_account_id} deleted")++ print("\n✓ Complete Vault encryption workflow test completed!")diff --git a/tests/integration/test_volcengine_kms_encryption_integration.py b/tests/integration/test_volcengine_kms_encryption_integration.pynew file mode 100644index 000000000..ea85e6fab--- /dev/null+++ b/tests/integration/test_volcengine_kms_encryption_integration.py@@ -0,0 +1,779 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0++"""+Volcengine KMS Encryption Integration Tests++Tests encryption functionality integrated with VikingFS and OpenViking service+using Volcengine KMS as the key provider.+Requires VOLCENGINE_ACCESS_KEY, VOLCENGINE_SECRET_KEY, and VOLCENGINE_KMS_KEY_ID env vars.+Run: pytest tests/integration/test_volcengine_kms_encryption_integration.py -v -m integration+"""++import secrets+import tempfile+from pathlib import Path++import pytest+import pytest_asyncio++from openviking import AsyncOpenViking+from openviking.crypto.encryptor import FileEncryptor+from openviking.crypto.providers import VolcengineKMSProvider+from openviking.server.api_keys import APIKeyManager+from openviking.service.core import OpenVikingService+from openviking_cli.session.user_id import UserIdentifier+from openviking_cli.utils.config.open_viking_config import OpenVikingConfigSingleton+from tests.integration.conftest import (+ VOLCENGINE_ACCESS_KEY,+ VOLCENGINE_KMS_KEY_ID,+ VOLCENGINE_KMS_REGION,+ VOLCENGINE_SECRET_KEY,+ requires_volcengine_kms,+)++pytestmark = [pytest.mark.integration, requires_volcengine_kms]+++class TestVolcengineKMSProvider:+ """Tests for VolcengineKMSProvider"""++ @pytest_asyncio.fixture(scope="module")+ async def volcengine_kms_provider(self):+ """+ Fixture that provides a VolcengineKMSProvider instance for testing.+ Requires valid Volcengine KMS credentials from environment variables.+ """+ provider = VolcengineKMSProvider(+ region=VOLCENGINE_KMS_REGION,+ access_key_id=VOLCENGINE_ACCESS_KEY,+ secret_access_key=VOLCENGINE_SECRET_KEY,+ key_id=VOLCENGINE_KMS_KEY_ID,+ )+ return provider++ @pytest.mark.asyncio+ async def test_initialize_provider(self, volcengine_kms_provider):+ """Test that VolcengineKMSProvider initializes correctly"""+ assert volcengine_kms_provider is not None+ assert volcengine_kms_provider.region == "cn-beijing"+ assert volcengine_kms_provider.key_id == "d926aa0d-deee-455a-9186-0d07e2e250e1"++ @pytest.mark.asyncio+ async def test_get_root_key(self, volcengine_kms_provider):+ """Test getting root key"""+ root_key = await volcengine_kms_provider.get_root_key()+ assert isinstance(root_key, bytes)+ assert len(root_key) > 0++ @pytest.mark.asyncio+ async def test_derive_account_key(self, volcengine_kms_provider):+ """Test deriving account key"""+ account_id = "test-account-volcengine"+ account_key = await volcengine_kms_provider.derive_account_key(account_id)+ assert isinstance(account_key, bytes)+ assert len(account_key) == 32 # Should be 256-bit key++ @pytest.mark.asyncio+ async def test_derive_different_account_keys(self, volcengine_kms_provider):+ """Test that different account IDs produce different keys"""+ account1 = "account-1-volc"+ account2 = "account-2-volc"++ key1 = await volcengine_kms_provider.derive_account_key(account1)+ key2 = await volcengine_kms_provider.derive_account_key(account2)++ assert key1 != key2++ @pytest.mark.asyncio+ async def test_derive_same_account_key(self, volcengine_kms_provider):+ """Test that same account ID produces same key"""+ account_id = "test-account-consistent"++ key1 = await volcengine_kms_provider.derive_account_key(account_id)+ key2 = await volcengine_kms_provider.derive_account_key(account_id)++ assert key1 == key2++ @pytest.mark.asyncio+ async def test_encrypt_decrypt_file_key(self, volcengine_kms_provider):+ """Test encrypting and decrypting a file key"""+ plaintext_key = secrets.token_bytes(32) # 256-bit key+ account_id = "test-account-encrypt"++ encrypted_key, iv = await volcengine_kms_provider.encrypt_file_key(+ plaintext_key, account_id+ )+ assert isinstance(encrypted_key, bytes)+ assert isinstance(iv, bytes)++ decrypted = await volcengine_kms_provider.decrypt_file_key(encrypted_key, iv, account_id)+ assert decrypted == plaintext_key++ @pytest.mark.asyncio+ async def test_encrypt_different_keys(self, volcengine_kms_provider):+ """Test that different plaintexts produce different ciphertexts"""+ key1 = secrets.token_bytes(32)+ key2 = secrets.token_bytes(32)+ account_id = "test-account-multi"++ encrypted1, iv1 = await volcengine_kms_provider.encrypt_file_key(key1, account_id)+ encrypted2, iv2 = await volcengine_kms_provider.encrypt_file_key(key2, account_id)++ assert encrypted1 != encrypted2++ @pytest.mark.asyncio+ async def test_full_roundtrip(self, volcengine_kms_provider):+ """Test complete roundtrip: derive account key, encrypt, decrypt"""+ account_id = "test-account-roundtrip"++ # Derive account key+ account_key = await volcengine_kms_provider.derive_account_key(account_id)+ assert isinstance(account_key, bytes)+ assert len(account_key) == 32++ # Generate and encrypt file key+ file_key = secrets.token_bytes(32)+ encrypted_file_key, iv = await volcengine_kms_provider.encrypt_file_key(+ file_key, account_id+ )++ # Decrypt file key+ decrypted_file_key = await volcengine_kms_provider.decrypt_file_key(+ encrypted_file_key, iv, account_id+ )+ assert decrypted_file_key == file_key+++@pytest_asyncio.fixture(scope="function")+async def volcengine_kms_config():+ """Fixture that provides Volcengine KMS encryption configuration"""+ return {+ "encryption": {+ "enabled": True,+ "provider": "volcengine_kms",+ "volcengine_kms": {+ "region": VOLCENGINE_KMS_REGION,+ "access_key": VOLCENGINE_ACCESS_KEY,+ "secret_key": VOLCENGINE_SECRET_KEY,+ "key_id": VOLCENGINE_KMS_KEY_ID,+ },+ }+ }+++@pytest_asyncio.fixture(scope="function")+async def volcengine_file_encryptor():+ """Fixture that provides a FileEncryptor with Volcengine KMS"""+ provider = VolcengineKMSProvider(+ region=VOLCENGINE_KMS_REGION,+ access_key_id=VOLCENGINE_ACCESS_KEY,+ secret_access_key=VOLCENGINE_SECRET_KEY,+ key_id=VOLCENGINE_KMS_KEY_ID,+ )+ return FileEncryptor(provider)+++@pytest_asyncio.fixture(scope="function")+async def openviking_client_with_volcengine_encryption(test_data_dir: Path, volcengine_kms_config):+ """Fixture that provides an OpenViking client with Volcengine KMS encryption"""+ await AsyncOpenViking.reset()+ OpenVikingConfigSingleton.reset_instance()++ if test_data_dir.exists():+ import shutil++ shutil.rmtree(test_data_dir)+ test_data_dir.mkdir(parents=True, exist_ok=True)++ config_dict = {}+ config_dict.update(volcengine_kms_config)+ config_dict["storage"] = {+ "workspace": str(test_data_dir / "workspace"),+ "vectordb": {"name": "test", "backend": "local", "project": "default"},+ }+ config_dict["embedding"] = {+ "dense": {+ "model": "doubao-embedding-vision-250615",+ "api_key": "2a2284d9-79a3-4978-b718-dfeffdda04ad",+ "api_base": "https://ark.cn-beijing.volces.com/api/v3",+ "dimension": 1024,+ "provider": "volcengine",+ }+ }++ OpenVikingConfigSingleton.initialize(config_dict=config_dict)++ client = AsyncOpenViking(path=str(test_data_dir))+ await client.initialize()++ yield client++ await client.close()+ await AsyncOpenViking.reset()+ OpenVikingConfigSingleton.reset_instance()+++class TestVolcengineKMSEncryptionBootstrap:+ """Tests for encryption module bootstrap with Volcengine KMS"""++ @pytest.mark.asyncio+ async def test_bootstrap_encryption_with_volcengine_kms(self, volcengine_kms_config, tmp_path):+ """Test bootstrapping encryption with Volcengine KMS"""+ from openviking.crypto.config import bootstrap_encryption++ config_dict = volcengine_kms_config+ config_dict["storage"] = {"workspace": str(tmp_path)}++ result = await bootstrap_encryption(config_dict)++ assert result is not None+ assert isinstance(result, FileEncryptor)++ @pytest.mark.asyncio+ async def test_file_encryptor_with_volcengine_kms(self, volcengine_file_encryptor):+ """Test FileEncryptor with Volcengine KMS"""+ account_id = "test_account_volcengine"+ plaintext = b"Test data for Volcengine KMS encryption"++ ciphertext = await volcengine_file_encryptor.encrypt(account_id, plaintext)+ assert ciphertext.startswith(b"OVE1")++ decrypted = await volcengine_file_encryptor.decrypt(account_id, ciphertext)+ assert decrypted == plaintext++ @pytest.mark.asyncio+ async def test_different_accounts_isolation(self, volcengine_file_encryptor):+ """Test that different accounts cannot decrypt each other's data"""+ account1 = "account_1_volc"+ account2 = "account_2_volc"+ plaintext = b"Sensitive data for account isolation test"++ # Encrypt with account1+ ciphertext = await volcengine_file_encryptor.encrypt(account1, plaintext)++ # Decrypt with account1 should work+ decrypted1 = await volcengine_file_encryptor.decrypt(account1, ciphertext)+ assert decrypted1 == plaintext++ # Decrypt with account2 should fail+ from openviking.crypto.exceptions import KeyMismatchError++ with pytest.raises(KeyMismatchError):+ await volcengine_file_encryptor.decrypt(account2, ciphertext)++ @pytest.mark.asyncio+ async def test_encrypt_empty_data(self, volcengine_file_encryptor):+ """Test encrypting and decrypting empty data"""+ account_id = "test_account_volc"+ plaintext = b""++ ciphertext = await volcengine_file_encryptor.encrypt(account_id, plaintext)+ assert ciphertext.startswith(b"OVE1")++ decrypted = await volcengine_file_encryptor.decrypt(account_id, ciphertext)+ assert decrypted == b""+++class TestVolcengineKMSEncryptionDisabled:+ """Tests for behavior when encryption is disabled"""++ @pytest_asyncio.fixture(scope="function")+ async def openviking_client_without_encryption(self, test_data_dir: Path):+ """Fixture that provides an OpenViking client without encryption"""+ await AsyncOpenViking.reset()+ OpenVikingConfigSingleton.reset_instance()++ if test_data_dir.exists():+ import shutil++ shutil.rmtree(test_data_dir)+ test_data_dir.mkdir(parents=True, exist_ok=True)++ # Create config dict with encryption disabled+ config_dict = {+ "encryption": {"enabled": False},+ "storage": {+ "workspace": str(test_data_dir / "workspace"),+ "vectordb": {"name": "test", "backend": "local", "project": "default"},+ },+ "embedding": {+ "dense": {+ "model": "doubao-embedding-vision-250615",+ "api_key": "2a2284d9-79a3-4978-b718-dfeffdda04ad",+ "api_base": "https://ark.cn-beijing.volces.com/api/v3",+ "dimension": 1024,+ "provider": "volcengine",+ }+ },+ }++ # Initialize config singleton+ OpenVikingConfigSingleton.initialize(config_dict=config_dict)++ client = AsyncOpenViking(path=str(test_data_dir))+ await client.initialize()++ yield client++ await client.close()+ await AsyncOpenViking.reset()+ OpenVikingConfigSingleton.reset_instance()++ @pytest.mark.asyncio+ async def test_read_write_without_encryption(+ self, openviking_client_without_encryption: AsyncOpenViking, tmp_path: Path+ ):+ """Test normal file operations when encryption is disabled"""+ client = openviking_client_without_encryption++ test_file = tmp_path / "normal_file.txt"+ test_content = "Normal content without encryption"+ test_file.write_text(test_content)++ result = await client.add_resource(+ path=str(test_file), reason="Normal operation test", wait=True+ )+ root_uri = result["root_uri"]++ # Get tree structure to find the actual file+ uris = await client.tree(root_uri)+ assert len(uris) > 0++ # Find the actual file (skip .abstract.md and .overview.md)+ found = False+ for data in uris:+ if not data["isDir"]:+ filename = data["name"]+ # Skip auto-generated files+ if filename not in [".abstract.md", ".overview.md"]:+ file_uri = data["uri"]+ content = await client.read(file_uri)+ assert content == test_content+ found = True+ break+ assert found, "Could not find the test file"+++class TestVikingFSEncryptionWithVolcengineKMS:+ """+ Complete VikingFS encryption tests using Volcengine KMS as key provider+ """++ ROOT_KEY = "test-root-key-for-volcengine-encryption-tests"++ @pytest_asyncio.fixture(scope="function")+ async def openviking_service_with_volcengine_encryption(self, test_data_dir: Path):+ """Fixture that provides OpenVikingService with Volcengine KMS encryption"""+ await AsyncOpenViking.reset()+ OpenVikingConfigSingleton.reset_instance()++ if test_data_dir.exists():+ import shutil++ shutil.rmtree(test_data_dir)+ test_data_dir.mkdir(parents=True, exist_ok=True)++ svc = OpenVikingService(+ path=str(test_data_dir / "viking"), user=UserIdentifier.the_default_user("test_user")+ )+ await svc.initialize()++ api_key_manager = APIKeyManager(+ root_key=self.ROOT_KEY, viking_fs=svc.viking_fs, encryption_enabled=True+ )+ await api_key_manager.load()++ yield {"service": svc, "api_key_manager": api_key_manager, "test_data_dir": test_data_dir}++ await svc.close()+ await AsyncOpenViking.reset()+ OpenVikingConfigSingleton.reset_instance()++ def _is_file_encrypted(self, file_path: Path) -> bool:+ """Check if file is encrypted (by "OVE1" header)"""+ if not file_path.exists():+ return False+ try:+ with open(file_path, "rb") as f:+ header = f.read(4)+ return header == b"OVE1"+ except Exception:+ return False++ async def _check_all_files_encrypted(+ self, svc, ctx, base_uri: str, print_paths: bool = True+ ) -> None:+ """Recursively check if all files in directory are encrypted"""+ agfs_client = svc._agfs_client++ async def _check_recursive(uri: str):+ try:+ entries = await svc.viking_fs.ls(uri, ctx=ctx)+ for entry in entries:+ entry_uri = entry["uri"]+ if entry["isDir"]:+ await _check_recursive(entry_uri)+ else:+ # Use VikingFS's _uri_to_path method to get file path+ agfs_path = svc.viking_fs._uri_to_path(entry_uri, ctx=ctx)+ raw_content = agfs_client.read(agfs_path)+ assert raw_content.startswith(b"OVE1"), f"File {entry_uri} is not encrypted"+ if print_paths:+ print(f"✓ File is encrypted: {entry_uri}")+ except Exception as e:+ if "not found" in str(e).lower():+ pass+ else:+ raise++ await _check_recursive(base_uri)++ @pytest.mark.asyncio+ async def test_bootstrap_encryption_with_volcengine_kms(+ self, openviking_service_with_volcengine_encryption+ ):+ """Test 1: Bootstrapping encryption with Volcengine KMS"""+ data = openviking_service_with_volcengine_encryption+ api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]++ account_id = "test-account-volcengine-bootstrap"+ admin_user_id = "admin"+ user_key = await api_key_manager.create_account(account_id, admin_user_id)++ assert user_key is not None+ assert len(user_key) == 64++ agfs_data_root = test_data_dir / "viking" / "viking"++ global_accounts_path = agfs_data_root / "_system" / "accounts.json"+ assert global_accounts_path.exists()+ assert self._is_file_encrypted(global_accounts_path)++ @pytest.mark.asyncio+ async def test_file_encryption_with_volcengine_kms(+ self, openviking_service_with_volcengine_encryption+ ):+ """Test 2: Encrypting files with Volcengine KMS"""+ data = openviking_service_with_volcengine_encryption+ svc = data["service"]+ api_key_manager = data["api_key_manager"]+ test_data_dir = data["test_data_dir"]++ account_id = "test-account-volcengine-file"+ admin_user_id = "admin"+ await api_key_manager.create_account(account_id, admin_user_id)++ from openviking.server.identity import RequestContext, Role++ user = UserIdentifier(account_id, admin_user_id, admin_user_id)+ ctx = RequestContext(user=user, role=Role.ADMIN)++ test_content = "Test content with Volcengine KMS encryption"+ with tempfile.NamedTemporaryFile(mode="w", suffix=".txt", delete=False) as f:+ f.write(test_content)+ temp_file_path = f.name++ try:+ resource = await svc.resources.add_resource(+ path=temp_file_path, reason="Test Volcengine KMS encryption", ctx=ctx, wait=True+ )+ resource["root_uri"]++ agfs_data_root = test_data_dir / "viking" / "viking"+ resources_dir = agfs_data_root / account_id / "resources"++ assert resources_dir.exists()++ await self._check_all_files_encrypted(svc, ctx, f"viking:///{account_id}/resources")++ finally:+ import os++ os.unlink(temp_file_path)++ @pytest.mark.asyncio+ async def test_resource_operations_with_encryption(+ self, openviking_service_with_volcengine_encryption, tmp_path+ ):+ """Test 4: Resource operations and encryption verification"""+ data = openviking_service_with_volcengine_encryption+ svc = data["service"]+ api_key_manager = data["api_key_manager"]++ # Create test account+ account_id = "test-account-resources"+ admin_user_id = "admin"+ await api_key_manager.create_account(account_id, admin_user_id)++ from openviking.server.identity import RequestContext, Role++ user = UserIdentifier(account_id, admin_user_id, admin_user_id)+ ctx = RequestContext(user=user, role=Role.ADMIN)++ # Create test resource file+ test_content = "This is test resource content for Volcengine KMS encryption"+ test_uri = f"viking://{account_id}/resources/test_file.txt"++ # Write file+ await svc.viking_fs.write_file(test_uri, test_content, ctx=ctx)++ # Verify read content is correct+ read_content = await svc.viking_fs.read_file(test_uri, ctx=ctx)+ assert read_content == test_content++ # Verify file is encrypted+ agfs_client = svc._agfs_client+ agfs_path = svc.viking_fs._uri_to_path(test_uri, ctx=ctx)+ raw_content = agfs_client.read(agfs_path)+ assert raw_content.startswith(b"OVE1")++ # Test various operations+ resources_dir_uri = f"viking://{account_id}/resources"++ # ls operation+ ls_entries = await svc.viking_fs.ls(resources_dir_uri, ctx=ctx)+ assert len(ls_entries) > 0++ # tree operation+ tree_entries = await svc.viking_fs.tree(resources_dir_uri, ctx=ctx)+ assert len(tree_entries) > 0++ # grep operation+ try:+ grep_result = await svc.viking_fs.grep(resources_dir_uri, "Volcengine", ctx=ctx)+ assert grep_result is not None+ except Exception as e:+ print(f"[WARNING] grep operation may not be supported: {e}")++ # abstract operation+ try:+ abstract = await svc.viking_fs.abstract(test_uri, ctx=ctx)+ assert abstract is not None+ assert "OVE1" not in abstract+ except Exception as e:+ print(f"[WARNING] abstract operation may not be supported: {e}")++ # overview operation+ try:+ overview = await svc.viking_fs.overview(test_uri, ctx=ctx)+ assert overview is not None+ assert "OVE1" not in overview+ except Exception as e:+ print(f"[WARNING] overview operation may not be supported: {e}")++ @pytest.mark.asyncio+ async def test_skill_operations_with_encryption(+ self, openviking_service_with_volcengine_encryption+ ):+ """Test 5: Skill operations and encryption verification"""+ data = openviking_service_with_volcengine_encryption+ svc = data["service"]+ api_key_manager = data["api_key_manager"]++ # Create test account+ account_id = "test-account-skills"+ admin_user_id = "admin"+ await api_key_manager.create_account(account_id, admin_user_id)++ from openviking.server.identity import RequestContext, Role++ user = UserIdentifier(account_id, admin_user_id, admin_user_id)+ ctx = RequestContext(user=user, role=Role.ADMIN)++ # Create skill directory and file+ skill_dir_uri = f"viking://{account_id}/agent/test-skill"+ skill_md_uri = f"{skill_dir_uri}/SKILL.md"++ # Create directory+ await svc.viking_fs.mkdir(skill_dir_uri, ctx=ctx)++ # Write skill file+ skill_content = """---+name: Test Skill+version: 1.0.0+description: Test skill for Volcengine KMS encryption+---++# Test Skill++This is a test skill for verifying Volcengine KMS encryption.+"""+ await svc.viking_fs.write_file(skill_md_uri, skill_content, ctx=ctx)++ # Verify read content is correct+ read_content = await svc.viking_fs.read_file(skill_md_uri, ctx=ctx)+ assert "Test Skill" in read_content++ # Verify file is encrypted+ agfs_client = svc._agfs_client+ agfs_path = svc.viking_fs._uri_to_path(skill_md_uri, ctx=ctx)+ raw_content = agfs_client.read(agfs_path)+ assert raw_content.startswith(b"OVE1")++ # Test various operations+ agent_dir_uri = f"viking://{account_id}/agent"++ # ls operation+ ls_entries = await svc.viking_fs.ls(agent_dir_uri, ctx=ctx)+ assert len(ls_entries) > 0++ # tree operation+ try:+ tree_entries = await svc.viking_fs.tree(agent_dir_uri, ctx=ctx)+ assert len(tree_entries) > 0+ except Exception as e:+ print(f"[WARNING] tree operation may not be supported: {e}")++ @pytest.mark.asyncio+ async def test_memory_operations_with_encryption(+ self, openviking_service_with_volcengine_encryption+ ):+ """Test 6: Memory operations and encryption verification"""+ data = openviking_service_with_volcengine_encryption+ svc = data["service"]+ api_key_manager = data["api_key_manager"]++ # Create test account+ account_id = "test-account-memories"+ user_id = "test-user"+ await api_key_manager.create_account(account_id, user_id)++ from openviking.server.identity import RequestContext, Role++ user = UserIdentifier(account_id, user_id, user_id)+ ctx = RequestContext(user=user, role=Role.USER)++ # Create memory directory and file+ memory_dir_uri = f"viking://{account_id}/user/{user_id}/memories"+ memory_file_uri = f"{memory_dir_uri}/preferences.md"++ # Create directory+ try:+ await svc.viking_fs.mkdir(memory_dir_uri, ctx=ctx)+ except Exception:+ pass # Directory may already exist++ # Write memory file+ memory_content = "# User Preferences\n\nTheme: dark\nLanguage: English"+ await svc.viking_fs.write_file(memory_file_uri, memory_content, ctx=ctx)++ # Verify read content is correct+ read_content = await svc.viking_fs.read_file(memory_file_uri, ctx=ctx)+ assert "User Preferences" in read_content++ # Verify file is encrypted+ agfs_client = svc._agfs_client+ agfs_path = svc.viking_fs._uri_to_path(memory_file_uri, ctx=ctx)+ raw_content = agfs_client.read(agfs_path)+ assert raw_content.startswith(b"OVE1")++ @pytest.mark.asyncio+ async def test_session_operations_with_encryption(+ self, openviking_service_with_volcengine_encryption+ ):+ """Test 7: Session operations and encryption verification"""+ data = openviking_service_with_volcengine_encryption+ svc = data["service"]+ api_key_manager = data["api_key_manager"]++ # Create test account+ account_id = "test-account-sessions"+ user_id = "test-user"+ await api_key_manager.create_account(account_id, user_id)++ from openviking.server.identity import RequestContext, Role++ user = UserIdentifier(account_id, user_id, user_id)+ ctx = RequestContext(user=user, role=Role.USER)++ # Create session+ session = await svc.sessions.create(ctx)+ session_id = session.session_id+ assert session_id is not None++ # Check if session directory files are encrypted+ session_dir_uri = f"viking://{account_id}/session"+ await self._check_all_files_encrypted(svc, ctx, session_dir_uri, print_paths=False)++ @pytest.mark.asyncio+ async def test_complete_encryption_workflow(+ self, openviking_service_with_volcengine_encryption, tmp_path+ ):+ """Test 8: Complete encryption workflow using Volcengine KMS as key provider"""+ data = openviking_service_with_volcengine_encryption+ svc = data["service"]+ api_key_manager = data["api_key_manager"]++ # Generate random account name+ import secrets++ random_suffix = secrets.token_hex(4)+ test_account_id = f"test-account-workflow-{random_suffix}"+ test_user_id = "admin"++ # Create account+ await api_key_manager.create_account(test_account_id, test_user_id)++ from openviking.server.identity import RequestContext, Role++ user = UserIdentifier(test_account_id, test_user_id, test_user_id)+ ctx = RequestContext(user=user, role=Role.ADMIN)++ print(f"\n✓ Created test account: {test_account_id}")++ # 1. Resource operations+ print("\n1. Testing resource operations...")+ resource_uri = f"viking://{test_account_id}/resources/test_workflow.txt"+ resource_content = "Test resource content for complete workflow"+ await svc.viking_fs.write_file(resource_uri, resource_content, ctx=ctx)+ read_content = await svc.viking_fs.read_file(resource_uri, ctx=ctx)+ assert read_content == resource_content+ print("✓ Resource operations test passed")++ # 2. Skill operations+ print("\n2. Testing skill operations...")+ skill_dir_uri = f"viking://{test_account_id}/agent/test-workflow-skill"+ skill_md_uri = f"{skill_dir_uri}/SKILL.md"+ await svc.viking_fs.mkdir(skill_dir_uri, ctx=ctx)+ skill_content = """---+name: Workflow Test Skill+version: 1.0.0+description: Test skill for complete workflow+---++# Workflow Test Skill+"""+ await svc.viking_fs.write_file(skill_md_uri, skill_content, ctx=ctx)+ read_skill = await svc.viking_fs.read_file(skill_md_uri, ctx=ctx)+ assert "Workflow Test Skill" in read_skill+ print("✓ Skill operations test passed")++ # 3. Memory operations+ print("\n3. Testing memory operations...")+ memory_dir_uri = f"viking://{test_account_id}/user/{test_user_id}/memories"+ memory_file_uri = f"{memory_dir_uri}/test_memory.md"+ try:+ await svc.viking_fs.mkdir(memory_dir_uri, ctx=ctx)+ except Exception:+ pass+ memory_content = "# Test Memory\n\nThis is a test memory for workflow"+ await svc.viking_fs.write_file(memory_file_uri, memory_content, ctx=ctx)+ read_memory = await svc.viking_fs.read_file(memory_file_uri, ctx=ctx)+ assert "Test Memory" in read_memory+ print("✓ Memory operations test passed")++ # 4. Session operations+ print("\n4. Testing session operations...")+ session = await svc.sessions.create(ctx)+ assert session.session_id is not None+ print("✓ Session operations test passed")++ # 5. Verify all files are encrypted+ print("\n5. Verifying all files are encrypted...")+ account_root_uri = f"viking://{test_account_id}"+ await self._check_all_files_encrypted(svc, ctx, account_root_uri, print_paths=False)+ print("✓ All files are encrypted")++ print("\n✓ Complete Volcengine KMS encryption workflow test passed!")diff --git a/tests/unit/crypto/test_providers.py b/tests/unit/crypto/test_local_provider.pysimilarity index 100%rename from tests/unit/crypto/test_providers.pyrename to tests/unit/crypto/test_local_provider.pydiff --git a/tests/unit/crypto/test_providers_mock.py b/tests/unit/crypto/test_providers_mock.pynew file mode 100644index 000000000..514b4df7e--- /dev/null+++ b/tests/unit/crypto/test_providers_mock.py@@ -0,0 +1,275 @@+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.+# SPDX-License-Identifier: Apache-2.0+"""+Mock-based unit tests for all KeyProvider implementations.++These tests do NOT require external services (Vault, Volcengine KMS) to run.+They use mocking to verify:+1. Envelope format compatibility+2. Provider-type byte round-tripping+3. API interaction correctness+4. Error handling+"""++import base64+import os+import secrets+import struct+import tempfile+from unittest.mock import AsyncMock, Mock, patch++import pytest++from openviking.crypto.encryptor import FileEncryptor+from openviking.crypto.providers import (+ PROVIDER_LOCAL,+ PROVIDER_VAULT,+ PROVIDER_VOLCENGINE,+ LocalFileProvider,+ VaultProvider,+ VolcengineKMSProvider,+)+++def local_file_provider():+ """Create a LocalFileProvider instance with temporary file."""+ with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:+ root_key = secrets.token_bytes(32)+ f.write(root_key.hex())+ temp_path = f.name+ os.chmod(temp_path, 0o600)++ try:+ provider = LocalFileProvider(key_file=temp_path)+ yield provider+ finally:+ if os.path.exists(temp_path):+ os.unlink(temp_path)+++def vault_mock_provider():+ """Create a VaultProvider with mocked methods."""+ provider = VaultProvider(+ addr="http://mock-vault:8200", token="mock-token", mount_path="transit"+ )++ # Mock all the methods that would call external services+ provider._get_client = AsyncMock()+ provider._ensure_transit_engine_enabled = AsyncMock()+ provider._ensure_root_key_exists = AsyncMock()++ yield provider+++def volcengine_mock_provider():+ """Create a VolcengineKMSProvider with mocked methods."""+ provider = VolcengineKMSProvider(+ access_key_id="mock-access-key",+ secret_access_key="mock-secret-key",+ region="cn-beijing",+ key_id="mock-key-id",+ )++ # Mock all the methods that would call external services+ provider._get_kms_client = AsyncMock()++ yield provider+++class TestEnvelopeFormat:+ """Tests for envelope format and provider-type byte round-tripping."""++ @pytest.mark.asyncio+ async def test_local_provider_envelope_format(self, local_file_provider):+ """Test envelope format with LocalFileProvider."""+ encryptor = FileEncryptor(local_file_provider)+ account_id = "test-account"+ plaintext = b"This is a test message"++ # Encrypt+ ciphertext = await encryptor.encrypt(account_id, plaintext)++ # Verify envelope starts with magic+ assert ciphertext.startswith(b"OVE1")++ # Verify provider type is in envelope+ assert ciphertext[5] == PROVIDER_LOCAL++ # Decrypt back+ decrypted = await encryptor.decrypt(account_id, ciphertext)+ assert decrypted == plaintext++ @pytest.mark.asyncio+ async def test_vault_provider_type_detection(self):+ """Test that VaultProvider type is correctly detected by FileEncryptor."""+ # Create a minimal mock provider that just needs type detection+ mock_provider = Mock(spec=VaultProvider)++ # FileEncryptor should detect it as Vault type+ encryptor = FileEncryptor(mock_provider)+ assert encryptor._provider_type == PROVIDER_VAULT++ @pytest.mark.asyncio+ async def test_volcengine_provider_type_detection(self):+ """Test that VolcengineKMSProvider type is correctly detected by FileEncryptor."""+ # Create a minimal mock provider that just needs type detection+ mock_provider = Mock(spec=VolcengineKMSProvider)++ # FileEncryptor should detect it as Volcengine type+ encryptor = FileEncryptor(mock_provider)+ assert encryptor._provider_type == PROVIDER_VOLCENGINE++ @pytest.mark.asyncio+ async def test_provider_type_roundtrip(self, local_file_provider):+ """Test that provider type is correctly preserved in envelope."""+ encryptor = FileEncryptor(local_file_provider)+ account_id = "test-account"+ plaintext = b"Roundtrip test"++ ciphertext = await encryptor.encrypt(account_id, plaintext)++ # Parse the envelope manually to verify provider type+ HEADER_SIZE = 12+ assert len(ciphertext) >= HEADER_SIZE++ (magic, version, provider_type, _, _, _) = struct.unpack(+ "!4sBBHHH", ciphertext[:HEADER_SIZE]+ )++ assert magic == b"OVE1"+ assert version == 0x01+ assert provider_type == PROVIDER_LOCAL+++class TestVaultProviderMock:+ """Tests for VaultProvider with mocked methods."""++ @pytest.mark.asyncio+ async def test_vault_encrypt_decrypt_with_mock(self, vault_mock_provider):+ """Test encrypt_file_key and decrypt_file_key can be mocked."""+ account_id = "test-account"+ plaintext_key = secrets.token_bytes(32)+ mock_ciphertext = b"vault:v1:encrypted_data"+ mock_iv = secrets.token_bytes(12)++ # Setup mock methods+ vault_mock_provider.encrypt_file_key = AsyncMock(return_value=(mock_ciphertext, mock_iv))+ vault_mock_provider.decrypt_file_key = AsyncMock(return_value=plaintext_key)++ # Encrypt+ encrypted, iv = await vault_mock_provider.encrypt_file_key(plaintext_key, account_id)+ assert encrypted == mock_ciphertext+ assert iv == mock_iv+ vault_mock_provider.encrypt_file_key.assert_called_once_with(plaintext_key, account_id)++ # Decrypt+ decrypted = await vault_mock_provider.decrypt_file_key(encrypted, iv, account_id)+ assert decrypted == plaintext_key+ vault_mock_provider.decrypt_file_key.assert_called_once_with(encrypted, iv, account_id)++ @pytest.mark.asyncio+ async def test_vault_provider_with_file_encryptor(self, vault_mock_provider):+ """Test FileEncryptor works with mocked VaultProvider."""+ # Setup mock methods+ test_file_key = secrets.token_bytes(32)+ test_iv = secrets.token_bytes(12)+ vault_mock_provider.get_root_key = AsyncMock(return_value=b"test_root_key")+ vault_mock_provider.derive_account_key = AsyncMock(return_value=test_file_key)+ vault_mock_provider.encrypt_file_key = AsyncMock(return_value=(b"encrypted_fk", test_iv))+ vault_mock_provider.decrypt_file_key = AsyncMock(return_value=test_file_key)++ encryptor = FileEncryptor(vault_mock_provider)+ account_id = "test-account"+ plaintext = b"Test with Vault mock"++ # Patch the AES encryption to simplify+ with (+ patch.object(+ encryptor, "_aes_gcm_encrypt", new=AsyncMock(return_value=b"encrypted_data")+ ),+ patch.object(encryptor, "_aes_gcm_decrypt", new=AsyncMock(return_value=plaintext)),+ ):+ ciphertext = await encryptor.encrypt(account_id, plaintext)+ assert ciphertext.startswith(b"OVE1")+ assert ciphertext[5] == PROVIDER_VAULT+++class TestVolcengineKMSProviderMock:+ """Tests for VolcengineKMSProvider with mocked methods."""++ @pytest.mark.asyncio+ async def test_volcengine_encrypt_decrypt_with_mock(self, volcengine_mock_provider):+ """Test encrypt_file_key and decrypt_file_key can be mocked."""+ account_id = "test-account"+ plaintext_key = secrets.token_bytes(32)+ mock_ciphertext = base64.b64encode(b"encrypted_data")+ mock_iv = secrets.token_bytes(12)++ # Setup mock methods+ volcengine_mock_provider.encrypt_file_key = AsyncMock(+ return_value=(mock_ciphertext, mock_iv)+ )+ volcengine_mock_provider.decrypt_file_key = AsyncMock(return_value=plaintext_key)++ # Encrypt+ encrypted, iv = await volcengine_mock_provider.encrypt_file_key(plaintext_key, account_id)+ assert encrypted == mock_ciphertext+ assert iv == mock_iv+ volcengine_mock_provider.encrypt_file_key.assert_called_once_with(plaintext_key, account_id)++ # Decrypt+ decrypted = await volcengine_mock_provider.decrypt_file_key(encrypted, iv, account_id)+ assert decrypted == plaintext_key+ volcengine_mock_provider.decrypt_file_key.assert_called_once_with(encrypted, iv, account_id)+++class TestCrossProviderEnvelope:+ """Tests for cross-provider envelope behavior."""++ @pytest.mark.asyncio+ async def test_different_provider_type_in_envelope(self):+ """Test that different providers set different type bytes in envelope."""+ # Create mock providers+ mock_local = Mock(spec=LocalFileProvider)+ mock_vault = Mock(spec=VaultProvider)+ mock_volcengine = Mock(spec=VolcengineKMSProvider)++ # Each should be detected as different types+ assert FileEncryptor(mock_local)._provider_type == PROVIDER_LOCAL+ assert FileEncryptor(mock_vault)._provider_type == PROVIDER_VAULT+ assert FileEncryptor(mock_volcengine)._provider_type == PROVIDER_VOLCENGINE++ @pytest.mark.asyncio+ async def test_same_provider_different_instance_can_decrypt(self, local_file_provider):+ """Test that same provider type (different instances) can decrypt each other's data."""+ # Create second local provider with the same key file+ # First, get the key from the first provider+ root_key = await local_file_provider.get_root_key()++ # Create second provider with same key+ with tempfile.NamedTemporaryFile(mode="w", delete=False) as f:+ f.write(root_key.hex())+ temp_path2 = f.name+ os.chmod(temp_path2, 0o600)++ try:+ provider2 = LocalFileProvider(key_file=temp_path2)++ encryptor1 = FileEncryptor(local_file_provider)+ encryptor2 = FileEncryptor(provider2)++ account_id = "test-account"+ plaintext = b"Same provider, different instance"++ # Encrypt with provider1+ ciphertext = await encryptor1.encrypt(account_id, plaintext)++ # Decrypt with provider2+ decrypted = await encryptor2.decrypt(account_id, ciphertext)+ assert decrypted == plaintext+ finally:+ if os.path.exists(temp_path2):+ os.unlink(temp_path2)