Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat(ovpack): support vector snapshots and consistency checks (#1965)

Qin Haojie5fdbfd4e49a2
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs
index e72cb80de..57e7a86ef 100644
--- a/crates/ov_cli/src/client.rs
+++ b/crates/ov_cli/src/client.rs
@@ -26,7 +26,15 @@ impl HttpClient {
extra_headers: Option<std::collections::HashMap<String, String>>,
) -> Self {
Self {
- base: BaseClient::new(base_url, api_key, agent_id, account, user, timeout_secs, extra_headers),
+ base: BaseClient::new(
+ base_url,
+ api_key,
+ agent_id,
+ account,
+ user,
+ timeout_secs,
+ extra_headers,
+ ),
}
}
@@ -108,16 +116,27 @@ impl HttpClient {
self.create_uploader().zip_directory(dir_path)
}
- fn zip_directory_with_progress(&self, dir_path: &Path, verbose: bool) -> Result<tempfile::NamedTempFile> {
- self.create_uploader().zip_directory_with_progress(dir_path, verbose)
+ fn zip_directory_with_progress(
+ &self,
+ dir_path: &Path,
+ verbose: bool,
+ ) -> Result<tempfile::NamedTempFile> {
+ self.create_uploader()
+ .zip_directory_with_progress(dir_path, verbose)
}
async fn upload_temp_file(&self, file_path: &Path) -> Result<String> {
self.create_uploader().upload_temp_file(file_path).await
}
- async fn upload_temp_file_with_progress(&self, file_path: &Path, verbose: bool) -> Result<String> {
- self.create_uploader().upload_temp_file_with_progress(file_path, verbose).await
+ async fn upload_temp_file_with_progress(
+ &self,
+ file_path: &Path,
+ verbose: bool,
+ ) -> Result<String> {
+ self.create_uploader()
+ .upload_temp_file_with_progress(file_path, verbose)
+ .await
}
// ============ Content Methods ============
@@ -165,12 +184,7 @@ impl HttpClient {
})
}
- pub async fn reindex(
- &self,
- uri: &str,
- mode: &str,
- wait: bool,
- ) -> Result<serde_json::Value> {
+ pub async fn reindex(&self, uri: &str, mode: &str, wait: bool) -> Result<serde_json::Value> {
let body = serde_json::json!({
"uri": uri,
"mode": mode,
@@ -179,6 +193,13 @@ impl HttpClient {
self.post("/api/v1/content/reindex", &body).await
}
+ pub async fn consistency(&self, uri: &str) -> Result<serde_json::Value> {
+ let body = serde_json::json!({
+ "uri": uri,
+ });
+ self.post("/api/v1/system/consistency", &body).await
+ }
+
/// Download file as raw bytes
pub async fn get_bytes(&self, uri: &str) -> Result<Vec<u8>> {
let url = format!("{}/api/v1/content/download", self.base.base_url);
@@ -443,7 +464,8 @@ impl HttpClient {
self.zip_directory(path_obj)?
};
let temp_file_id = if show_progress {
- self.upload_temp_file_with_progress(zip_file.path(), verbose).await?
+ self.upload_temp_file_with_progress(zip_file.path(), verbose)
+ .await?
} else {
self.upload_temp_file(zip_file.path()).await?
};
@@ -465,15 +487,19 @@ impl HttpClient {
"watch_interval": watch_interval,
}));
- let dynamic_timeout = TimeoutConfig::for_resource_processing().calculate(zip_file.path())?;
- self.base.post_with_timeout("/api/v1/resources", &body, dynamic_timeout).await
+ let dynamic_timeout =
+ TimeoutConfig::for_resource_processing().calculate(zip_file.path())?;
+ self.base
+ .post_with_timeout("/api/v1/resources", &body, dynamic_timeout)
+ .await
} else if path_obj.is_file() {
let source_name = path_obj
.file_name()
.and_then(|n| n.to_str())
.map(|s| s.to_string());
let temp_file_id = if show_progress {
- self.upload_temp_file_with_progress(path_obj, verbose).await?
+ self.upload_temp_file_with_progress(path_obj, verbose)
+ .await?
} else {
self.upload_temp_file(path_obj).await?
};
@@ -495,8 +521,11 @@ impl HttpClient {
"watch_interval": watch_interval,
}));
- let dynamic_timeout = TimeoutConfig::for_resource_processing().calculate(path_obj)?;
- self.base.post_with_timeout("/api/v1/resources", &body, dynamic_timeout).await
+ let dynamic_timeout =
+ TimeoutConfig::for_resource_processing().calculate(path_obj)?;
+ self.base
+ .post_with_timeout("/api/v1/resources", &body, dynamic_timeout)
+ .await
} else {
let body = build_body(serde_json::json!({
"path": path,
@@ -555,8 +584,11 @@ impl HttpClient {
"wait": wait,
"timeout": timeout,
});
- let dynamic_timeout = TimeoutConfig::for_resource_processing().calculate(zip_file.path())?;
- self.base.post_with_timeout("/api/v1/skills", &body, dynamic_timeout).await
+ let dynamic_timeout =
+ TimeoutConfig::for_resource_processing().calculate(zip_file.path())?;
+ self.base
+ .post_with_timeout("/api/v1/skills", &body, dynamic_timeout)
+ .await
} else if path_obj.is_file() {
let temp_file_id = self.upload_temp_file(path_obj).await?;
@@ -565,8 +597,11 @@ impl HttpClient {
"wait": wait,
"timeout": timeout,
});
- let dynamic_timeout = TimeoutConfig::for_resource_processing().calculate(path_obj)?;
- self.base.post_with_timeout("/api/v1/skills", &body, dynamic_timeout).await
+ let dynamic_timeout =
+ TimeoutConfig::for_resource_processing().calculate(path_obj)?;
+ self.base
+ .post_with_timeout("/api/v1/skills", &body, dynamic_timeout)
+ .await
} else {
let body = serde_json::json!({
"data": data,
@@ -707,9 +742,15 @@ impl HttpClient {
Ok(final_path.to_string_lossy().to_string())
}
- pub async fn export_ovpack(&self, uri: &str, to: &str) -> Result<String> {
+ pub async fn export_ovpack(
+ &self,
+ uri: &str,
+ to: &str,
+ include_vectors: bool,
+ ) -> Result<String> {
let body = serde_json::json!({
"uri": uri,
+ "include_vectors": include_vectors,
});
let base_name = uri
.trim_end_matches('/')
@@ -720,10 +761,10 @@ impl HttpClient {
.await
}
- pub async fn backup_ovpack(&self, to: &str) -> Result<String> {
+ pub async fn backup_ovpack(&self, to: &str, include_vectors: bool) -> Result<String> {
self.download_pack(
"/api/v1/pack/backup",
- serde_json::json!({}),
+ serde_json::json!({"include_vectors": include_vectors}),
to,
"openviking-backup",
)
@@ -735,6 +776,7 @@ impl HttpClient {
file_path: &str,
parent: &str,
on_conflict: Option<&str>,
+ vector_mode: Option<&str>,
) -> Result<serde_json::Value> {
let file_path_obj = Path::new(file_path);
@@ -754,6 +796,7 @@ impl HttpClient {
"temp_file_id": temp_file_id,
"parent": parent,
"on_conflict": conflict_policy,
+ "vector_mode": vector_mode.unwrap_or("auto"),
});
self.post("/api/v1/pack/import", &body).await
}
@@ -762,6 +805,7 @@ impl HttpClient {
&self,
file_path: &str,
on_conflict: Option<&str>,
+ vector_mode: Option<&str>,
) -> Result<serde_json::Value> {
let file_path_obj = Path::new(file_path);
@@ -780,6 +824,7 @@ impl HttpClient {
let body = serde_json::json!({
"temp_file_id": temp_file_id,
"on_conflict": conflict_policy,
+ "vector_mode": vector_mode.unwrap_or("auto"),
});
self.post("/api/v1/pack/restore", &body).await
}
@@ -1035,11 +1080,15 @@ mod tests {
let headers = client.build_headers();
assert_eq!(
- headers.get("X-API-Key").and_then(|value| value.to_str().ok()),
+ headers
+ .get("X-API-Key")
+ .and_then(|value| value.to_str().ok()),
Some("test-key")
);
assert_eq!(
- headers.get("X-Custom-Header").and_then(|value| value.to_str().ok()),
+ headers
+ .get("X-Custom-Header")
+ .and_then(|value| value.to_str().ok()),
Some("custom-value")
);
}
diff --git a/crates/ov_cli/src/commands/pack.rs b/crates/ov_cli/src/commands/pack.rs
index 5ab409b9f..4c1c52d46 100644
--- a/crates/ov_cli/src/commands/pack.rs
+++ b/crates/ov_cli/src/commands/pack.rs
@@ -6,10 +6,11 @@ pub async fn export(
client: &HttpClient,
uri: &str,
to: &str,
+ include_vectors: bool,
format: OutputFormat,
compact: bool,
) -> Result<()> {
- let file_path = client.export_ovpack(uri, to).await?;
+ let file_path = client.export_ovpack(uri, to, include_vectors).await?;
// Output success message with the file path
let result = serde_json::json!({
@@ -24,10 +25,11 @@ pub async fn export(
pub async fn backup(
client: &HttpClient,
to: &str,
+ include_vectors: bool,
format: OutputFormat,
compact: bool,
) -> Result<()> {
- let file_path = client.backup_ovpack(to).await?;
+ let file_path = client.backup_ovpack(to, include_vectors).await?;
let result = serde_json::json!({
"file": file_path,
@@ -43,11 +45,12 @@ pub async fn import(
file_path: &str,
target: &str,
on_conflict: Option<&str>,
+ vector_mode: Option<&str>,
format: OutputFormat,
compact: bool,
) -> Result<()> {
let result = client
- .import_ovpack(file_path, target, on_conflict)
+ .import_ovpack(file_path, target, on_conflict, vector_mode)
.await?;
output_success(&result, format, compact);
Ok(())
@@ -57,10 +60,13 @@ pub async fn restore(
client: &HttpClient,
file_path: &str,
on_conflict: Option<&str>,
+ vector_mode: Option<&str>,
format: OutputFormat,
compact: bool,
) -> Result<()> {
- let result = client.restore_ovpack(file_path, on_conflict).await?;
+ let result = client
+ .restore_ovpack(file_path, on_conflict, vector_mode)
+ .await?;
output_success(&result, format, compact);
Ok(())
}
diff --git a/crates/ov_cli/src/commands/system.rs b/crates/ov_cli/src/commands/system.rs
index 96dea7803..8cebadea3 100644
--- a/crates/ov_cli/src/commands/system.rs
+++ b/crates/ov_cli/src/commands/system.rs
@@ -26,6 +26,48 @@ pub async fn status(client: &HttpClient, output_format: OutputFormat, compact: b
Ok(())
}
+pub async fn consistency(
+ client: &HttpClient,
+ uri: &str,
+ output_format: OutputFormat,
+ compact: bool,
+) -> Result<()> {
+ let response: serde_json::Value = client.consistency(uri).await?;
+ if matches!(output_format, OutputFormat::Table) {
+ output_consistency_table(&response, compact);
+ } else {
+ output_success(&response, output_format, compact);
+ }
+ Ok(())
+}
+
+fn output_consistency_table(response: &serde_json::Value, compact: bool) {
+ let summary = json!({
+ "ok": response.get("ok").and_then(|v| v.as_bool()).unwrap_or(false),
+ "expected_count": response.get("expected_count").and_then(|v| v.as_u64()).unwrap_or(0),
+ "missing_record_count": response
+ .get("missing_record_count")
+ .and_then(|v| v.as_u64())
+ .unwrap_or(0),
+ "missing_records_truncated": response
+ .get("missing_records_truncated")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false),
+ });
+ output_success(&summary, OutputFormat::Table, compact);
+
+ let Some(missing_records) = response.get("missing_records").and_then(|v| v.as_array()) else {
+ return;
+ };
+ if missing_records.is_empty() {
+ return;
+ }
+
+ println!();
+ println!("missing_records");
+ output_success(missing_records, OutputFormat::Table, compact);
+}
+
pub async fn health(
client: &HttpClient,
output_format: OutputFormat,
diff --git a/crates/ov_cli/src/handlers.rs b/crates/ov_cli/src/handlers.rs
index 10ff8a7db..831928b1f 100644
--- a/crates/ov_cli/src/handlers.rs
+++ b/crates/ov_cli/src/handlers.rs
@@ -1,10 +1,10 @@
+use crate::CliContext;
+use crate::PrivacyCommands;
use crate::client;
use crate::commands;
use crate::config::merge_csv_options;
use crate::error::{Error, Result};
use crate::tui;
-use crate::CliContext;
-use crate::PrivacyCommands;
pub async fn handle_add_resource(
mut path: String,
@@ -59,12 +59,20 @@ pub async fn handle_add_resource(
// Check that only one of --to, --parent, or --parent-auto-create is set
let mut exclusive_count = 0;
- if to.is_some() { exclusive_count += 1; }
- if parent.is_some() { exclusive_count += 1; }
- if parent_auto_create.is_some() { exclusive_count += 1; }
+ if to.is_some() {
+ exclusive_count += 1;
+ }
+ if parent.is_some() {
+ exclusive_count += 1;
+ }
+ if parent_auto_create.is_some() {
+ exclusive_count += 1;
+ }
if exclusive_count > 1 {
- eprintln!("Error: Cannot specify more than one of --to, --parent, or --parent-auto-create at the same time.");
+ eprintln!(
+ "Error: Cannot specify more than one of --to, --parent, or --parent-auto-create at the same time."
+ );
std::process::exit(1);
}
@@ -160,20 +168,41 @@ pub async fn handle_unlink(from_uri: String, to_uri: String, ctx: CliContext) ->
commands::relations::unlink(&client, &from_uri, &to_uri, ctx.output_format, ctx.compact).await
}
-pub async fn handle_export(uri: String, to: String, ctx: CliContext) -> Result<()> {
+pub async fn handle_export(
+ uri: String,
+ to: String,
+ include_vectors: bool,
+ ctx: CliContext,
+) -> Result<()> {
let client = ctx.get_client();
- commands::pack::export(&client, &uri, &to, ctx.output_format, ctx.compact).await
+ commands::pack::export(
+ &client,
+ &uri,
+ &to,
+ include_vectors,
+ ctx.output_format,
+ ctx.compact,
+ )
+ .await
}
-pub async fn handle_backup(to: String, ctx: CliContext) -> Result<()> {
+pub async fn handle_backup(to: String, include_vectors: bool, ctx: CliContext) -> Result<()> {
let client = ctx.get_client();
- commands::pack::backup(&client, &to, ctx.output_format, ctx.compact).await
+ commands::pack::backup(
+ &client,
+ &to,
+ include_vectors,
+ ctx.output_format,
+ ctx.compact,
+ )
+ .await
}
pub async fn handle_import(
file_path: String,
target_uri: String,
on_conflict: Option<String>,
+ vector_mode: Option<String>,
ctx: CliContext,
) -> Result<()> {
let client = ctx.get_client();
@@ -182,6 +211,7 @@ pub async fn handle_import(
&file_path,
&target_uri,
on_conflict.as_deref(),
+ vector_mode.as_deref(),
ctx.output_format,
ctx.compact,
)
@@ -191,6 +221,7 @@ pub async fn handle_import(
pub async fn handle_restore(
file_path: String,
on_conflict: Option<String>,
+ vector_mode: Option<String>,
ctx: CliContext,
) -> Result<()> {
let client = ctx.get_client();
@@ -198,6 +229,7 @@ pub async fn handle_restore(
&client,
&file_path,
on_conflict.as_deref(),
+ vector_mode.as_deref(),
ctx.output_format,
ctx.compact,
)
@@ -219,6 +251,9 @@ pub async fn handle_system(cmd: SystemCommands, ctx: CliContext) -> Result<()> {
let _ = commands::system::health(&client, ctx.output_format, ctx.compact).await?;
Ok(())
}
+ SystemCommands::Consistency { uri } => {
+ commands::system::consistency(&client, &uri, ctx.output_format, ctx.compact).await
+ }
SystemCommands::Crypto { action } => commands::crypto::handle_crypto(action).await,
}
}
@@ -356,8 +391,22 @@ pub async fn handle_admin(cmd: AdminCommands, ctx: CliContext) -> Result<()> {
)
.await
}
- AdminCommands::ListUsers { account_id, limit, name, role } => {
- commands::admin::list_users(&client, &account_id, limit, name, role, ctx.output_format, ctx.compact).await
+ AdminCommands::ListUsers {
+ account_id,
+ limit,
+ name,
+ role,
+ } => {
+ commands::admin::list_users(
+ &client,
+ &account_id,
+ limit,
+ name,
+ role,
+ ctx.output_format,
+ ctx.compact,
+ )
+ .await
}
AdminCommands::ListAgents { account_id } => {
commands::admin::list_agents(&client, &account_id, ctx.output_format, ctx.compact).await
@@ -424,14 +473,16 @@ pub async fn handle_privacy(cmd: PrivacyCommands, ctx: CliContext) -> Result<()>
PrivacyCommands::Get {
category,
target_key,
- } => commands::privacy::get_current(
- &client,
- &category,
- &target_key,
- ctx.output_format,
- ctx.compact,
- )
- .await,
+ } => {
+ commands::privacy::get_current(
+ &client,
+ &category,
+ &target_key,
+ ctx.output_format,
+ ctx.compact,
+ )
+ .await
+ }
PrivacyCommands::Upsert {
category,
target_key,
@@ -458,46 +509,52 @@ pub async fn handle_privacy(cmd: PrivacyCommands, ctx: CliContext) -> Result<()>
PrivacyCommands::Versions {
category,
target_key,
- } => commands::privacy::list_versions(
- &client,
- &category,
- &target_key,
- ctx.output_format,
- ctx.compact,
- )
- .await,
+ } => {
+ commands::privacy::list_versions(
+ &client,
+ &category,
+ &target_key,
+ ctx.output_format,
+ ctx.compact,
+ )
+ .await
+ }
PrivacyCommands::Version {
category,
target_key,
version,
- } => commands::privacy::get_version(
- &client,
- &category,
- &target_key,
- version,
- ctx.output_format,
- ctx.compact,
- )
- .await,
+ } => {
+ commands::privacy::get_version(
+ &client,
+ &category,
+ &target_key,
+ version,
+ ctx.output_format,
+ ctx.compact,
+ )
+ .await
+ }
PrivacyCommands::Activate {
category,
target_key,
version,
- } => commands::privacy::activate(
- &client,
- &category,
- &target_key,
- version,
- ctx.output_format,
- ctx.compact,
- )
- .await,
+ } => {
+ commands::privacy::activate(
+ &client,
+ &category,
+ &target_key,
+ version,
+ ctx.output_format,
+ ctx.compact,
+ )
+ .await
+ }
}
}
use crate::ConfigCommands;
-use crate::config::Config;
use crate::base_client::BaseClient;
+use crate::config::Config;
use crate::output;
pub async fn handle_config(cmd: ConfigCommands, _ctx: CliContext) -> Result<()> {
@@ -518,19 +575,15 @@ pub async fn handle_config(cmd: ConfigCommands, _ctx: CliContext) -> Result<()>
}
Err(e) => Err(Error::Config(e.to_string())),
},
- ConfigCommands::SetupCli => {
- handle_setup_cli().await
- },
- ConfigCommands::Switch => {
- handle_config_switch().await
- },
+ ConfigCommands::SetupCli => handle_setup_cli().await,
+ ConfigCommands::Switch => handle_config_switch().await,
}
}
/// Interactive setup for CLI configuration
async fn handle_setup_cli() -> Result<()> {
- use rustyline::DefaultEditor;
use colored::Colorize;
+ use rustyline::DefaultEditor;
println!("{}", "Welcome to OpenViking CLI Setup!".bold().green());
println!();
@@ -541,7 +594,8 @@ async fn handle_setup_cli() -> Result<()> {
Err(_) => Config::default(),
};
- let mut rl = DefaultEditor::new().map_err(|e| Error::Config(format!("Failed to initialize input editor: {}", e)))?;
+ let mut rl = DefaultEditor::new()
+ .map_err(|e| Error::Config(format!("Failed to initialize input editor: {}", e)))?;
// Step 1: Get server URL
let default_url = config.url.clone();
@@ -597,7 +651,10 @@ async fn handle_setup_cli() -> Result<()> {
// Step 4: Save config
println!();
let config_path = crate::config::default_config_path()?;
- println!("{}", format!("Saving configuration to: {}", config_path.to_string_lossy()).blue());
+ println!(
+ "{}",
+ format!("Saving configuration to: {}", config_path.to_string_lossy()).blue()
+ );
config.save_default()?;
// Step 5: Optionally save a named backup
@@ -608,15 +665,30 @@ async fn handle_setup_cli() -> Result<()> {
let config_name = name_input.trim().to_string();
if let Some(parent) = config_path.parent() {
let backup_path = parent.join(format!("ovcli.conf.{}", config_name));
- std::fs::copy(&config_path, &backup_path).map_err(|e| Error::Config(format!("Failed to save configuration backup: {}", e)))?;
- println!("{}", format!("✓ Configuration saved as backup: {}", backup_path.to_string_lossy()).green());
+ std::fs::copy(&config_path, &backup_path).map_err(|e| {
+ Error::Config(format!("Failed to save configuration backup: {}", e))
+ })?;
+ println!(
+ "{}",
+ format!(
+ "✓ Configuration saved as backup: {}",
+ backup_path.to_string_lossy()
+ )
+ .green()
+ );
}
}
println!();
println!("{}", "✓ Setup complete!".bold().green());
- println!("{}", "You can now use the 'ov' command to interact with OpenViking.".dimmed());
- println!("{}", "Use 'ov config switch' to switch between saved configurations.".dimmed());
+ println!(
+ "{}",
+ "You can now use the 'ov' command to interact with OpenViking.".dimmed()
+ );
+ println!(
+ "{}",
+ "Use 'ov config switch' to switch between saved configurations.".dimmed()
+ );
Ok(())
}
@@ -630,20 +702,26 @@ async fn probe_health(base_url: &str, timeout_secs: f64) -> Result<(bool, bool)>
match result {
Ok(value) => {
- let healthy = value.get("healthy").and_then(|v| v.as_bool()).unwrap_or(false);
+ let healthy = value
+ .get("healthy")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
Ok((healthy, false))
}
Err(Error::Api(msg)) => {
// If we get auth-related errors, auth is required
- if msg.contains("401") || msg.contains("403") || msg.contains("unauthorized") || msg.contains("forbidden") || msg.contains("AuthenticationError") {
+ if msg.contains("401")
+ || msg.contains("403")
+ || msg.contains("unauthorized")
+ || msg.contains("forbidden")
+ || msg.contains("AuthenticationError")
+ {
Ok((false, true))
} else {
Ok((false, false))
}
}
- Err(_) => {
- Ok((false, true))
- }
+ Err(_) => Ok((false, true)),
}
}
@@ -653,7 +731,9 @@ async fn handle_config_switch() -> Result<()> {
// Step 1: Find all available configurations
let config_path = crate::config::default_config_path()?;
- let config_dir = config_path.parent().ok_or_else(|| Error::Config("Could not determine config directory".to_string()))?;
+ let config_dir = config_path
+ .parent()
+ .ok_or_else(|| Error::Config("Could not determine config directory".to_string()))?;
let mut configs = Vec::new();
@@ -678,7 +758,10 @@ async fn handle_config_switch() -> Result<()> {
if configs.is_empty() {
println!("{}", "No saved configurations found.".yellow());
- println!("{}", "Use 'ov config setup-cli' and save a named configuration first.".dimmed());
+ println!(
+ "{}",
+ "Use 'ov config setup-cli' and save a named configuration first.".dimmed()
+ );
return Ok(());
}
@@ -695,11 +778,18 @@ async fn handle_config_switch() -> Result<()> {
}
println!();
- println!("{}", "Enter '<number>' to use a configuration (e.g. '2')".dimmed());
- println!("{}", "Or enter 'del<number>' to delete a configuration (e.g. 'del2')".dimmed());
+ println!(
+ "{}",
+ "Enter '<number>' to use a configuration (e.g. '2')".dimmed()
+ );
+ println!(
+ "{}",
+ "Or enter 'del<number>' to delete a configuration (e.g. 'del2')".dimmed()
+ );
println!("{}", "Press Enter without input to cancel".dimmed());
- let mut rl = rustyline::DefaultEditor::new().map_err(|e| Error::Config(format!("Failed to initialize input editor: {}", e)))?;
+ let mut rl = rustyline::DefaultEditor::new()
+ .map_err(|e| Error::Config(format!("Failed to initialize input editor: {}", e)))?;
loop {
let input = rl.readline("> ").unwrap_or_default();
@@ -712,17 +802,31 @@ async fn handle_config_switch() -> Result<()> {
}
// Check for delete command
- if let Some(num_str) = input.strip_prefix("del").or_else(|| input.strip_prefix("Del")) {
+ if let Some(num_str) = input
+ .strip_prefix("del")
+ .or_else(|| input.strip_prefix("Del"))
+ {
if let Ok(idx) = num_str.parse::<usize>() {
if idx >= 1 && idx <= configs.len() {
let idx = idx - 1;
let (name, path, _) = &configs[idx];
println!();
- println!("{}", format!("Are you sure you want to delete configuration '{}'? (y/N)", name).yellow());
+ println!(
+ "{}",
+ format!(
+ "Are you sure you want to delete configuration '{}'? (y/N)",
+ name
+ )
+ .yellow()
+ );
let confirm = rl.readline("> ").unwrap_or_default();
- if confirm.trim().eq_ignore_ascii_case("y") || confirm.trim().eq_ignore_ascii_case("yes") {
- std::fs::remove_file(path).map_err(|e| Error::Config(format!("Failed to delete configuration: {}", e)))?;
+ if confirm.trim().eq_ignore_ascii_case("y")
+ || confirm.trim().eq_ignore_ascii_case("yes")
+ {
+ std::fs::remove_file(path).map_err(|e| {
+ Error::Config(format!("Failed to delete configuration: {}", e))
+ })?;
println!("{}", format!("✓ Configuration '{}' deleted.", name).green());
} else {
println!("{}", "Deletion cancelled.".dimmed());
@@ -744,21 +848,39 @@ async fn handle_config_switch() -> Result<()> {
// First, backup current config as .bak
if config_path.exists() {
let backup_path = config_dir.join("ovcli.conf.bak");
- std::fs::copy(&config_path, &backup_path).map_err(|e| Error::Config(format!("Failed to backup current config: {}", e)))?;
+ std::fs::copy(&config_path, &backup_path).map_err(|e| {
+ Error::Config(format!("Failed to backup current config: {}", e))
+ })?;
}
// Copy selected config to main config
- std::fs::copy(source_path, &config_path).map_err(|e| Error::Config(format!("Failed to switch configuration: {}", e)))?;
+ std::fs::copy(source_path, &config_path)
+ .map_err(|e| Error::Config(format!("Failed to switch configuration: {}", e)))?;
println!();
- println!("{}", format!("✓ Switched to configuration '{}'", name).bold().green());
- println!("{}", format!("Configuration saved to: {}", config_path.to_string_lossy()).dimmed());
+ println!(
+ "{}",
+ format!("✓ Switched to configuration '{}'", name)
+ .bold()
+ .green()
+ );
+ println!(
+ "{}",
+ format!("Configuration saved to: {}", config_path.to_string_lossy()).dimmed()
+ );
return Ok(());
}
}
- println!("{}", format!("Invalid selection. Please enter a number between 1 and {}.", configs.len()).yellow());
+ println!(
+ "{}",
+ format!(
+ "Invalid selection. Please enter a number between 1 and {}.",
+ configs.len()
+ )
+ .yellow()
+ );
}
}
@@ -810,12 +932,7 @@ pub async fn handle_write(
.await
}
-pub async fn handle_reindex(
- uri: String,
- mode: String,
- wait: bool,
- ctx: CliContext,
-) -> Result<()> {
+pub async fn handle_reindex(uri: String, mode: String, wait: bool, ctx: CliContext) -> Result<()> {
let client = ctx.get_client();
commands::content::reindex(&client, &uri, &mode, wait, ctx.output_format, ctx.compact).await
}
@@ -1069,7 +1186,12 @@ pub async fn handle_grep(
.await
}
-pub async fn handle_glob(pattern: String, uri: String, node_limit: i32, ctx: CliContext) -> Result<()> {
+pub async fn handle_glob(
+ pattern: String,
+ uri: String,
+ node_limit: i32,
+ ctx: CliContext,
+) -> Result<()> {
let params = vec![
format!("--uri={}", uri),
format!("-n {}", node_limit),
@@ -1104,7 +1226,10 @@ pub async fn handle_tui(uri: String, ctx: CliContext) -> Result<()> {
println!("Connecting to {}...", ctx.config.url);
match client.get::<serde_json::Value>("/health", &[]).await {
Ok(value) => {
- let healthy = value.get("healthy").and_then(|v| v.as_bool()).unwrap_or(false);
+ let healthy = value
+ .get("healthy")
+ .and_then(|v| v.as_bool())
+ .unwrap_or(false);
if !healthy {
println!("Warning: Server reports unhealthy status");
}
diff --git a/crates/ov_cli/src/main.rs b/crates/ov_cli/src/main.rs
index 1a2f54b99..170da19f7 100644
--- a/crates/ov_cli/src/main.rs
+++ b/crates/ov_cli/src/main.rs
@@ -9,10 +9,10 @@ mod tui;
mod utils;
use clap::{ArgAction, Parser, Subcommand};
-use std::ffi::OsString;
use config::Config;
use error::Result;
use output::OutputFormat;
+use std::ffi::OsString;
/// CLI context shared across commands
#[derive(Debug, Clone)]
@@ -493,11 +493,17 @@ enum Commands {
uri: String,
/// Output .ovpack file path
to: String,
+ /// Include dense vector snapshot when compatible metadata is available
+ #[arg(long, default_value_t = false)]
+ include_vectors: bool,
},
/// [Data] Back up public OpenViking scopes as a restore-only .ovpack
Backup {
/// Output .ovpack file path
to: String,
+ /// Include dense vector snapshot when compatible metadata is available
+ #[arg(long, default_value_t = false)]
+ include_vectors: bool,
},
/// [Data] Import .ovpack into target URI
Import {
@@ -508,6 +514,9 @@ enum Commands {
/// Conflict policy: fail, overwrite, or skip
#[arg(long, value_parser = ["fail", "overwrite", "skip"])]
on_conflict: Option<String>,
+ /// Vector handling: auto restores compatible snapshots, recompute ignores them, require fails if unavailable
+ #[arg(long, value_parser = ["auto", "recompute", "require"])]
+ vector_mode: Option<String>,
},
/// [Data] Restore a backup .ovpack to original public scope roots
Restore {
@@ -516,6 +525,9 @@ enum Commands {
/// Conflict policy: fail, overwrite, or skip
#[arg(long, value_parser = ["fail", "overwrite", "skip"])]
on_conflict: Option<String>,
+ /// Vector handling: auto restores compatible snapshots, recompute ignores them, require fails if unavailable
+ #[arg(long, value_parser = ["auto", "recompute", "require"])]
+ vector_mode: Option<String>,
},
// --- Interactive Tools ---
/// [Interactive] Interactive TUI file explorer
@@ -603,9 +615,7 @@ impl Commands {
/// Returns true if this is an admin command that supports --sudo
fn is_admin_command(&self) -> bool {
match self {
- Self::Admin { .. }
- | Self::System { .. }
- | Self::Reindex { .. } => true,
+ Self::Admin { .. } | Self::System { .. } | Self::Reindex { .. } => true,
_ => false,
}
}
@@ -641,6 +651,11 @@ enum SystemCommands {
Status,
/// Quick health check
Health,
+ /// Check filesystem and vector-index consistency for a URI subtree
+ Consistency {
+ /// Viking URI to check
+ uri: String,
+ },
/// Cryptographic key management commands
Crypto {
#[command(subcommand)]
@@ -917,7 +932,11 @@ fn preprocess_privacy_upsert_key_flags(args: Vec<OsString>) -> Vec<OsString> {
if args[cmd_idx].to_string_lossy() != "privacy" {
return args;
}
- if args.get(cmd_idx + 1).map(|s| s.to_string_lossy().to_string()) != Some("upsert".to_string()) {
+ if args
+ .get(cmd_idx + 1)
+ .map(|s| s.to_string_lossy().to_string())
+ != Some("upsert".to_string())
+ {
return args;
}
@@ -1009,7 +1028,9 @@ async fn main() {
// Check if --sudo is used but root_api_key is not configured
if ctx.sudo && ctx.config.root_api_key.is_none() {
- eprintln!("Error: --sudo requires root_api_key to be configured in ~/.openviking/ovcli.conf");
+ eprintln!(
+ "Error: --sudo requires root_api_key to be configured in ~/.openviking/ovcli.conf"
+ );
std::process::exit(2);
}
@@ -1066,20 +1087,29 @@ async fn main() {
to_uris,
reason,
} => handlers::handle_link(from_uri, to_uris, reason, ctx).await,
- Commands::Unlink { from_uri, to_uri } => handlers::handle_unlink(from_uri, to_uri, ctx).await,
- Commands::Export { uri, to } => handlers::handle_export(uri, to, ctx).await,
- Commands::Backup { to } => handlers::handle_backup(to, ctx).await,
+ Commands::Unlink { from_uri, to_uri } => {
+ handlers::handle_unlink(from_uri, to_uri, ctx).await
+ }
+ Commands::Export {
+ uri,
+ to,
+ include_vectors,
+ } => handlers::handle_export(uri, to, include_vectors, ctx).await,
+ Commands::Backup {
+ to,
+ include_vectors,
+ } => handlers::handle_backup(to, include_vectors, ctx).await,
Commands::Import {
file_path,
target_uri,
on_conflict,
- } => {
- handlers::handle_import(file_path, target_uri, on_conflict, ctx).await
- }
+ vector_mode,
+ } => handlers::handle_import(file_path, target_uri, on_conflict, vector_mode, ctx).await,
Commands::Restore {
file_path,
on_conflict,
- } => handlers::handle_restore(file_path, on_conflict, ctx).await,
+ vector_mode,
+ } => handlers::handle_restore(file_path, on_conflict, vector_mode, ctx).await,
Commands::Wait { timeout } => {
let client = ctx.get_client();
commands::system::wait(&client, timeout, ctx.output_format, ctx.compact).await
@@ -1202,13 +1232,12 @@ async fn main() {
} else {
"replace".to_string()
};
- handlers::handle_write(uri, content, from_file, effective_mode, wait, timeout, ctx).await
+ handlers::handle_write(uri, content, from_file, effective_mode, wait, timeout, ctx)
+ .await
+ }
+ Commands::Reindex { uri, mode, wait } => {
+ handlers::handle_reindex(uri, mode, wait, ctx).await
}
- Commands::Reindex {
- uri,
- mode,
- wait,
- } => handlers::handle_reindex(uri, mode, wait, ctx).await,
Commands::Get { uri, local_path } => handlers::handle_get(uri, local_path, ctx).await,
Commands::Find {
query,
@@ -1402,7 +1431,10 @@ mod tests {
"--no-vectorize",
]);
- assert!(result.is_err(), "removed import vectorize flag should not parse");
+ assert!(
+ result.is_err(),
+ "removed import vectorize flag should not parse"
+ );
}
#[test]
@@ -1415,7 +1447,10 @@ mod tests {
"--force",
]);
- assert!(result.is_err(), "removed import force flag should not parse");
+ assert!(
+ result.is_err(),
+ "removed import force flag should not parse"
+ );
}
#[test]
diff --git a/docs/en/api/01-overview.md b/docs/en/api/01-overview.md
index 7a86389c0..27d03e1ed 100644
--- a/docs/en/api/01-overview.md
+++ b/docs/en/api/01-overview.md
@@ -348,6 +348,7 @@ Below are all HTTP API endpoints provided by OpenViking, grouped by functional m
| GET | `/metrics` | Prometheus metrics export | Optional |
| GET | `/api/v1/system/status` | System status | Required |
| POST | `/api/v1/system/wait` | Wait for processing | Required |
+| POST | `/api/v1/system/consistency` | Filesystem/vector-index consistency check | Required |
### Resources
diff --git a/docs/en/api/03-filesystem.md b/docs/en/api/03-filesystem.md
index d0b6d67fc..f4bf64ef1 100644
--- a/docs/en/api/03-filesystem.md
+++ b/docs/en/api/03-filesystem.md
@@ -994,10 +994,12 @@ Packages all resources under the specified URI into a `.ovpack` file for backup
5. Return as file stream
**Format Notes**:
-- The exported ZIP contains `<root>/_._ovpack_manifest.json`, the escaped ZIP name for `.ovpack_manifest.json`.
+- The exported ZIP stores user content unchanged under `<root>/files/` and internal metadata under `<root>/_ovpack/`.
+- The manifest is stored at `<root>/_ovpack/manifest.json`.
- `entries[].path` is relative to the exported root; `""` means the root directory itself.
- File entries include `size` and `sha256`; `content_sha256` covers the sorted file list of `path`, `size`, and `sha256`.
-- Raw embedding vectors and runtime fields such as `created_at`, `updated_at`, and `active_count` are not exported.
+- `_ovpack/index_records.jsonl` stores portable index scalar fields. With `include_vectors=true`, `_ovpack/dense.f32` stores a pure-dense float32 vector snapshot plus embedding metadata; vector indexes whose `VectorIndex.IndexType` is hybrid do not support vector snapshot export.
+- Runtime fields such as `id`, `uri`, `account_id`, `created_at`, `updated_at`, and `active_count` are regenerated in the target environment and are not restored from the package.
- OVPack does not add package-size, file-count, or directory-depth limits; the practical limit comes from ZIP, the storage backend, and the runtime environment.
**Code Entry Points**:
@@ -1012,6 +1014,7 @@ Packages all resources under the specified URI into a `.ovpack` file for backup
| Parameter | Type | Required | Default | Description |
|-----------|------|----------|---------|-------------|
| uri | string | Yes | - | Viking URI to export |
+| include_vectors | boolean | No | false | Include a pure-dense vector snapshot; hybrid index types are rejected |
**Permission Requirements**: ROOT or ADMIN
@@ -1029,7 +1032,8 @@ curl -X POST http://localhost:1933/api/v1/pack/export \
-H "Content-Type: application/json" \
-H "X-API-Key: your-admin-key" \
-d '{
- "uri": "viking://resources/my-project/"
+ "uri": "viking://resources/my-project/",
+ "include_vectors": false
}' \
--output my-project.ovpack
```
@@ -1051,6 +1055,9 @@ client.initialize()
```bash
# Export resource
ov export viking://resources/my-project/ ./exports/my-project.ovpack
+
+# Export with a dense vector snapshot
+ov export viking://resources/my-project/ ./exports/my-project.ovpack --include-vectors
```
**Response Example**
@@ -1088,11 +1095,14 @@ Imports a `.ovpack` file to a specified location for restoring or migrating data
| temp_file_id | string | Yes | - | Temporary upload file ID (obtained via [temp_upload](02-resources.md#temp_upload)) |
| parent | string | Yes | - | Target parent URI (import to this location) |
| on_conflict | string | No | fail | Conflict policy: `fail`, `overwrite`, or `skip` |
+| vector_mode | string | No | auto | Vector handling: `auto`, `recompute`, or `require` |
**Permission Requirements**: ROOT or ADMIN
**Behavior Notes**:
-- Imports always rebuild vectors in the target environment. The API no longer accepts `vectorize` or `force`.
+- The API no longer accepts `vectorize` or `force`.
+- `vector_mode=auto` restores a compatible dense snapshot when present, otherwise recomputes vectors. `recompute` always ignores package vectors. `require` fails unless a compatible dense snapshot is present.
+- Dense snapshot compatibility checks compare embedding provider, model, input mode, query/document parameters, and dimensions.
- Session imports restore session files and do not trigger vectorization.
- `on_conflict=fail` returns a structured `409 CONFLICT` when the target root already exists.
- `on_conflict=overwrite` replaces the existing target root. `on_conflict=skip` keeps the existing target root and returns it without writing package contents. `skip` is root-level, not file-level.
@@ -1100,7 +1110,7 @@ Imports a `.ovpack` file to a specified location for restoring or migrating data
- Packages with manifest entries are rejected if content files or directories are missing, extra files or directories are present, file sizes differ, per-file `sha256` differs, or `content_sha256` is missing or differs.
- Packages whose manifest `format_version` is not the current supported version are rejected.
- `.abstract.md` and `.overview.md` are restored as semantic sidecars. `.relations.json` and OVPack internals are excluded.
-- Manifest `context_type` must match the final import path semantics.
+- Manifest `context_type`, when present in index scalar metadata, must match the final import path semantics.
- Top-level scope packages such as `viking://resources/` must be imported to `viking://`.
- OVPack does not add import package-size, file-count, or directory-depth limits; the practical limit comes from ZIP, the storage backend, and the runtime environment.
@@ -1129,7 +1139,8 @@ curl -X POST http://localhost:1933/api/v1/pack/import \
-d "{
\"temp_file_id\": \"$TEMP_FILE_ID\",
\"parent\": \"viking://resources/imported/\",
- \"on_conflict\": \"overwrite\"
+ \"on_conflict\": \"overwrite\",
+ \"vector_mode\": \"auto\"
}"
```
@@ -1153,6 +1164,9 @@ ov import ./exports/my-project.ovpack viking://resources/imported/
# Explicit conflict policy
ov import ./exports/my-project.ovpack viking://resources/imported/ --on-conflict overwrite
+
+# Require restoring a compatible dense vector snapshot
+ov import ./exports/my-project.ovpack viking://resources/imported/ --vector-mode require
```
**Response Example**
@@ -1190,7 +1204,8 @@ ov import ./exports/my-project.ovpack viking://resources/imported/ --on-conflict
Back up public scope roots as a restore-only `.ovpack` file. The backup includes
`resources`, `user`, `agent`, and `session`; it excludes internal runtime data
-such as `temp` and `queue`.
+such as `temp` and `queue`. Set `include_vectors=true` to include compatible
+pure-dense vector snapshots; hybrid index types reject vector snapshot export.
```
POST /api/v1/pack/backup
@@ -1198,7 +1213,9 @@ POST /api/v1/pack/backup
```bash
curl -X POST http://localhost:1933/api/v1/pack/backup \
+ -H "Content-Type: application/json" \
-H "X-API-Key: your-admin-key" \
+ -d '{"include_vectors":false}' \
--output openviking-backup.ovpack
```
@@ -1206,6 +1223,7 @@ CLI:
```bash
ov backup ./backups/openviking.ovpack
+ov backup ./backups/openviking.ovpack --include-vectors
```
---
@@ -1213,13 +1231,14 @@ ov backup ./backups/openviking.ovpack
### restore_ovpack
Restore a backup package created by `backup_ovpack` to the original public scope
-roots. Regular import rejects backup packages. Non-session content is
-re-vectorized after restore; session files are restored without vectorization.
+roots. Regular import rejects backup packages. Vector handling follows
+`vector_mode`; session files are restored without vectorization.
| Parameter | Type | Required | Default | Description |
|-----------|------|----------|---------|-------------|
| temp_file_id | string | Yes | - | Temporary upload file ID |
| on_conflict | string | No | fail | Conflict policy: `fail`, `overwrite`, or `skip` |
+| vector_mode | string | No | auto | Vector handling: `auto`, `recompute`, or `require` |
```
POST /api/v1/pack/restore
@@ -1237,13 +1256,14 @@ TEMP_FILE_ID=$(
curl -X POST http://localhost:1933/api/v1/pack/restore \
-H "Content-Type: application/json" \
-H "X-API-Key: your-admin-key" \
- -d "{\"temp_file_id\":\"$TEMP_FILE_ID\",\"on_conflict\":\"overwrite\"}"
+ -d "{\"temp_file_id\":\"$TEMP_FILE_ID\",\"on_conflict\":\"overwrite\",\"vector_mode\":\"auto\"}"
```
CLI:
```bash
ov restore ./backups/openviking.ovpack --on-conflict overwrite
+ov restore ./backups/openviking.ovpack --on-conflict overwrite --vector-mode require
```
---
diff --git a/docs/en/api/07-system.md b/docs/en/api/07-system.md
index 1dca700ff..e46bc41c5 100644
--- a/docs/en/api/07-system.md
+++ b/docs/en/api/07-system.md
@@ -165,6 +165,85 @@ ov system status
---
+### consistency
+
+#### 1. API Implementation Overview
+
+Check filesystem/vector-index consistency for a URI subtree. This is a general
+data consistency API for debugging missing index records, failed vector snapshot
+exports, and related issues. It is not an OVPack-private API;
+`ov export --include-vectors` and `ov backup --include-vectors` reuse the same
+check.
+
+The response returns only a summary and missing records. It does not return the
+full expected-record list. `missing_records` includes at most the first 20
+records; `missing_records_truncated` is `true` when more missing records exist.
+
+**Code Entry Points**:
+- `openviking/server/routers/system.py:check_consistency` - HTTP route
+- `openviking_cli/client/sync_http.py:SyncHTTPClient.check_consistency` - SDK entry
+- `crates/ov_cli/src/commands/system.rs:consistency` - CLI command
+
+#### 2. Interface and Parameters
+
+| Parameter | Type | Required | Default | Description |
+|-----------|------|----------|---------|-------------|
+| uri | string | Yes | - | Viking URI subtree to check |
+
+#### 3. Usage Examples
+
+**HTTP API**
+
+```
+POST /api/v1/system/consistency
+Content-Type: application/json
+```
+
+```bash
+curl -X POST http://localhost:1933/api/v1/system/consistency \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{"uri":"viking://resources/my-project"}'
+```
+
+**Python SDK**
+
+```python
+report = client.check_consistency("viking://resources/my-project")
+print(report["ok"])
+print(report["missing_records"])
+```
+
+**CLI**
+
+```bash
+ov system consistency viking://resources/my-project
+```
+
+**Response Example**
+
+```json
+{
+ "status": "ok",
+ "result": {
+ "ok": false,
+ "expected_count": 3,
+ "missing_record_count": 1,
+ "missing_records_truncated": false,
+ "missing_records": [
+ {
+ "uri": "viking://resources/my-project/README.md",
+ "path": "README.md",
+ "level": 2,
+ "key": "README.md#level=2"
+ }
+ ]
+ }
+}
+```
+
+---
+
### wait_processed
#### 1. API Implementation Overview
diff --git a/docs/en/guides/09-ovpack.md b/docs/en/guides/09-ovpack.md
index e802e95da..f2aced4e1 100644
--- a/docs/en/guides/09-ovpack.md
+++ b/docs/en/guides/09-ovpack.md
@@ -1,118 +1,264 @@
# OVPack Import and Export
-OVPack is OpenViking's packaging format for exporting/importing context subtrees (e.g., resources and memories) for backup, migration, and sharing.
+OVPack is OpenViking's recoverable content package format for migrating or
+backing up public content trees under `viking://`. It stores file content,
+semantic sidecar files, portable index scalar fields, and optional dense vector
+snapshots.
+
+OVPack is not a raw ZIP copy and is not a trusted publishing format. Import
+validates the manifest, file list, directory list, and checksums so package
+content cannot drift from the manifest. If an attacker can rewrite both content
+and manifest, rely on external signatures, secure transport, and access control.
+
+## Supported Scope
+
+Regular `export/import` handles one package root:
+
+- `viking://resources/...`
+- `viking://user/...`
+- `viking://agent/...`
+- `viking://session/...`
+
+Full migration uses the separate `backup/restore` flow. It packages public
+scope roots together:
+
+- `viking://resources`
+- `viking://user`
+- `viking://agent`
+- `viking://session`
+
+Internal or runtime data such as `temp`, `queue`, `upload`, lock files, watch
+control files, and `.relations.json` are outside the OVPack migration scope.
## Quick Start
-### Export Resources
+### Export and Import a Resource Directory
-Export OpenViking resources to an `.ovpack` file.
+```bash
+ov export viking://resources/my-project ./exports/my-project.ovpack
+ov import ./exports/my-project.ovpack viking://resources/imported/
+```
+
+The import target is the parent directory, not the final root. If the package
+root is `my-project`, the imported URI is:
+
+```text
+viking://resources/imported/my-project
+```
+
+Overwrite an existing root:
-**CLI**
```bash
-openviking export viking://resources/my-project/ ./exports/my-project.ovpack
+ov import ./exports/my-project.ovpack viking://resources/imported/ --on-conflict overwrite
```
-**Python SDK**
-```python
-from openviking import AsyncOpenViking
+### Export Vector Snapshots
-async def export_example():
- client = AsyncOpenViking()
- await client.initialize()
- try:
- exported_path = await client.export_ovpack(
- uri="viking://resources/my-project/",
- to="./exports/my-project.ovpack"
- )
- print(f"Export successful: {exported_path}")
- finally:
- await client.close()
+By default, export does not store dense vectors. Import recomputes vectors in
+the target environment:
+
+```bash
+ov export viking://resources/my-project ./exports/my-project.ovpack
+ov import ./exports/my-project.ovpack viking://resources/imported/
```
-### Import Resources
+If the export and import environments use the same embedding configuration, you
+can explicitly include a dense vector snapshot:
+
+```bash
+ov export viking://resources/my-project ./exports/my-project.ovpack --include-vectors
+ov import ./exports/my-project.ovpack viking://resources/imported/ --vector-mode auto
+```
-Import an `.ovpack` file into OpenViking.
+`--vector-mode` controls how import handles package vectors:
+
+| Value | Behavior |
+| --- | --- |
+| `auto` | Default. Restore a dense snapshot when present and embedding metadata is compatible; otherwise recompute vectors. |
+| `recompute` | Ignore package dense snapshots and always recompute vectors. |
+| `require` | Require a compatible dense snapshot. Missing, incomplete, model-mismatched, or dimension-mismatched snapshots fail import. |
+
+Compatibility checks compare the package embedding provider, model, input,
+query/document parameters, and dimensions with the current environment. OVPack
+vector snapshots currently support pure dense indexes only. If the underlying
+`VectorIndex.IndexType` is hybrid, `--include-vectors` fails the export. When
+importing into a hybrid-index environment, `auto` recomputes vectors and
+`require` fails.
+
+Before exporting a dense vector snapshot, OpenViking runs a data consistency
+check. It verifies that content expected to be in the vector index already has
+matching index records. Missing records fail the export so the package does not
+carry an incomplete index snapshot.
+
+You can call the consistency check directly when debugging data state:
-**CLI**
```bash
-# Basic import
-openviking import ./exports/my-project.ovpack viking://resources/imported/
+ov system consistency viking://resources/my-project
+```
+
+The API returns only a summary and at most 20 missing records. It does not return
+the full expected-record list. When `--include-vectors` export fails, error
+details include only one missing key to keep logs small.
-# Explicit conflict policy
-openviking import ./exports/my-project.ovpack viking://resources/imported/ --on-conflict overwrite
+Python SDK:
+
+```python
+report = await client.check_consistency("viking://resources/my-project")
+print(report["ok"], report["missing_records"])
```
-**Python SDK**
+HTTP API:
+
+```bash
+curl -X POST http://localhost:1933/api/v1/system/consistency \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-admin-key" \
+ -d '{"uri":"viking://resources/my-project"}'
+```
+
+### Full Backup and Restore
+
+Do not use `export viking://` for full migration. Use a backup package:
+
+```bash
+ov backup ./backups/openviking.ovpack
+ov restore ./backups/openviking.ovpack --on-conflict overwrite
+```
+
+Backup packages can only be restored with `restore`; regular `import` rejects
+them.
+
+## Python SDK
+
```python
from openviking import AsyncOpenViking
-async def import_example():
+
+async def migrate_project():
client = AsyncOpenViking()
await client.initialize()
try:
+ await client.export_ovpack(
+ uri="viking://resources/my-project",
+ to="./exports/my-project.ovpack",
+ include_vectors=False,
+ )
+
imported_uri = await client.import_ovpack(
file_path="./exports/my-project.ovpack",
parent="viking://resources/imported/",
- on_conflict="overwrite"
+ on_conflict="overwrite",
+ vector_mode="auto",
)
- print(f"Import successful: {imported_uri}")
+ print(imported_uri)
await client.wait_processed()
finally:
await client.close()
```
-**HTTP API**
+Full backup:
+
+```python
+await client.backup_ovpack("./backups/openviking.ovpack", include_vectors=True)
+await client.restore_ovpack(
+ "./backups/openviking.ovpack",
+ on_conflict="overwrite",
+ vector_mode="auto",
+)
+```
+
+## HTTP API
+
+HTTP export returns a file stream directly. HTTP import and restore first upload
+the local `.ovpack`, then call the pack endpoint with `temp_file_id`.
+
+Export:
+
+```bash
+curl -X POST http://localhost:1933/api/v1/pack/export \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-admin-key" \
+ -d '{"uri":"viking://resources/my-project","include_vectors":false}' \
+ --output my-project.ovpack
+```
+
+Import:
+
```bash
-# Step 1: Upload the local ovpack file
-# This uses local temporary storage by default.
-# Add: -F "upload_mode=shared" only when you explicitly need distributed shared temporary uploads.
-# Python HTTP client / CLI users can instead set ovcli.conf: upload.mode = "shared".
TEMP_FILE_ID=$(
curl -sS -X POST http://localhost:1933/api/v1/resources/temp_upload \
- -H "X-API-Key: your-key" \
- -F 'file=@./exports/my-project.ovpack' \
- | jq -r '.result.temp_file_id'
+ -H "X-API-Key: your-admin-key" \
+ -F "file=@./exports/my-project.ovpack" \
+ | jq -r ".result.temp_file_id"
)
-# Step 2: Import using temp_file_id
curl -X POST http://localhost:1933/api/v1/pack/import \
-H "Content-Type: application/json" \
- -H "X-API-Key: your-key" \
+ -H "X-API-Key: your-admin-key" \
-d "{
\"temp_file_id\": \"$TEMP_FILE_ID\",
\"parent\": \"viking://resources/imported/\",
- \"on_conflict\": \"overwrite\"
+ \"on_conflict\": \"overwrite\",
+ \"vector_mode\": \"auto\"
}"
```
-## Format Notes
+Full backup:
-OVPack v2 files are standard ZIP archives. Each package contains an OpenViking
-manifest at `<root>/_._ovpack_manifest.json`; this is the ZIP-escaped form of the
-hidden filename `.ovpack_manifest.json`.
+```bash
+curl -X POST http://localhost:1933/api/v1/pack/backup \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-admin-key" \
+ -d '{"include_vectors":true}' \
+ --output openviking-backup.ovpack
+```
+
+## Conflict Policy
+
+`on_conflict` only applies when the import root already exists.
+
+| Value | Behavior |
+| --- | --- |
+| `fail` | Default. Return `409 CONFLICT` when the target root exists. |
+| `overwrite` | Delete the existing root, then write package content and rebuild index state. |
+| `skip` | Return the existing root URI without writing package content. |
+
+`skip` is a root-level skip, not a file-level merge.
+
+## Package Layout
+
+OVPack v2 is a standard ZIP archive with one package root directory:
+
+```text
+my-project/
+my-project/files/
+my-project/files/notes.txt
+my-project/files/.abstract.md
+my-project/files/.overview.md
+my-project/_ovpack/
+my-project/_ovpack/index_records.jsonl
+my-project/_ovpack/dense.f32 # only with --include-vectors and exportable vectors
+my-project/_ovpack/manifest.json
+```
-The manifest records `kind`, `format_version`, the exported root, content
-entries, and portable vector scalar metadata. In `entries`, `path` is relative to
-the exported root. An empty path (`""`) means the root directory itself, for
-example `my-project/`.
+`files/` stores user content with the same relative paths used by OpenViking.
+Dotfiles are no longer escaped with `_._`. `_ovpack/` stores OVPack internal
+metadata and is not imported as user content.
-Example manifest for a package exported from `viking://resources/demo/`:
+The manifest stores package structure, file checksums, and checksums for
+internal index files. It does not inline per-file index records:
```json
{
"kind": "openviking.ovpack",
"format_version": 2,
"root": {
- "name": "demo",
- "uri": "viking://resources/demo",
+ "name": "my-project",
+ "uri": "viking://resources/my-project",
"scope": "resources"
},
"entries": [
- {
- "path": "",
- "kind": "directory"
- },
+ {"path": "", "kind": "directory"},
{
"path": "notes.txt",
"kind": "file",
@@ -121,215 +267,213 @@ Example manifest for a package exported from `viking://resources/demo/`:
}
],
"content_sha256": "b2a6e9582119c7510d68e3446de3e71a486934bf450d68f65596259ed1cf7997",
- "vectors": {}
+ "index": {
+ "records": {
+ "path": "_ovpack/index_records.jsonl",
+ "count": 2,
+ "sha256": "..."
+ },
+ "dense": {
+ "path": "_ovpack/dense.f32",
+ "count": 1,
+ "dtype": "float32",
+ "byte_order": "little",
+ "dimensions": 1024,
+ "sha256": "...",
+ "embedding": {
+ "provider": "volcengine",
+ "model": "doubao-embedding-vision-251215",
+ "input": "multimodal",
+ "dimensions": 1024
+ }
+ }
+ }
}
```
-For packages with manifest entries, import validates the ZIP file and directory
-set, per-file `size`, per-file `sha256`, and top-level `content_sha256` before
-writing any resources. Missing, extra, or modified content entries, and missing
-or mismatched v2 `content_sha256`, are rejected. This is an integrity check for the package
-contents, not a signature or authentication mechanism if both the manifest and
-content can be rewritten.
-
-Raw embedding vectors are not exported. Runtime fields such as `created_at`,
-`updated_at`, and `active_count` are also not exported; imports rebuild vectors
-and runtime state in the target environment. Session packages restore session
-files and do not trigger vectorization. Packages without a manifest are rejected
-by default because OpenViking cannot verify their file set or content checksums.
-Packages with a manifest validate `kind` and `format_version`, and packages
-whose format version is not the current supported version are rejected.
-`.abstract.md` and `.overview.md` are restored as semantic sidecar files;
-`.relations.json` and OVPack internals are excluded.
-
-Manifest scalar `context_type`, when present, is treated as exported scalar
-metadata only. Import compatibility is based on the source and target URI scopes;
-the final `context_type` is derived again from the target URI during
-vectorization.
-
-Top-level scope packages such as `viking://resources/`, `viking://user/`,
-`viking://agent/`, and `viking://session/` must be imported to `viking://`.
-Regular import also requires the manifest root scope to match the final target
-root scope. Structured scopes (`user`, `agent`, and `session`) cannot be imported
-in a way that changes the root depth, such as importing a session package into a
-concrete session URI and creating `session/sess_123/sess_123`.
-Full backups use the separate `backup` / `restore` interface, not regular
-import parent semantics.
-OVPack itself does not add package-size, file-count, or directory-depth limits;
-the practical limit comes from ZIP, the storage backend, and the runtime
-environment.
-
-## Memory Import and Export
-
-OpenViking memories are stored under fixed directory structures:
-
-- User memories: `viking://user/{user_space}/memories/`
-- Agent memories: `viking://agent/{agent_id}/memories/` or `viking://agent/{agent_id}/user/{user_id}/memories/`
-
-When migrating memories with OVPack, you must import the `.ovpack` into the parent of the corresponding space (not an arbitrary directory). Otherwise you may end up with paths like `.../memories/memories/...`, and OpenViking will not be able to access and use them as memories.
-
-### Export/Import User Memories (CLI)
+`entries[].path == ""` means the package root directory itself. Nested paths are
+stored as relative paths:
-```bash
-# Export the whole user memories subtree
-openviking export viking://user/default/memories/ ./exports/user-memories.ovpack
-
-# Import into the user space root (imports to viking://user/default/memories/)
-openviking import ./exports/user-memories.ovpack viking://user/default/ --on-conflict overwrite
+```json
+[
+ {"path": "", "kind": "directory"},
+ {"path": "docs", "kind": "directory"},
+ {"path": "docs/a.md", "kind": "file", "size": 12, "sha256": "..."}
+]
```
-### Export/Import Agent Memories (CLI)
+`index_records.jsonl` contains one index record per line and can describe files,
+directories, and the root directory:
-```bash
-# isolate_agent_scope_by_user = false
-openviking export viking://agent/default/memories/ ./exports/agent-memories.ovpack
-openviking import ./exports/agent-memories.ovpack viking://agent/default/ --on-conflict overwrite
-
-# isolate_agent_scope_by_user = true
-openviking export viking://agent/default/user/alice/memories/ ./exports/agent-memories.ovpack
-openviking import ./exports/agent-memories.ovpack viking://agent/default/user/alice/ --on-conflict overwrite
+```jsonl
+{"record_id":"r000001","path":"","kind":"directory","level":0,"text":"root abstract","scalars":{"abstract":"root abstract","context_type":"resource","level":0}}
+{"record_id":"r000002","path":"notes.txt","kind":"file","level":2,"scalars":{"abstract":"note summary","tags":"demo"},"vector":{"dense":{"offset":0,"dimensions":1024}}}
```
-### Export/Import Memories (Python SDK)
-
-```python
-from openviking import AsyncOpenViking
+`dense.f32` is a contiguous little-endian float32 array. The
+`vector.dense.offset` in `index_records.jsonl` is a float offset, not a byte
+offset. In the example above, `offset=0, dimensions=1024` means reading 1024
+float values from float offset 0.
-async def export_import_user_memories():
- client = AsyncOpenViking()
- await client.initialize()
- try:
- await client.export_ovpack(
- uri="viking://user/default/memories/",
- to="./exports/user-memories.ovpack",
- )
+## Index Fields
- await client.import_ovpack(
- file_path="./exports/user-memories.ovpack",
- parent="viking://user/default/",
- on_conflict="overwrite",
- )
- finally:
- await client.close()
+By default, export stores portable index scalar fields:
-async def export_import_agent_memories():
- client = AsyncOpenViking()
- await client.initialize()
- try:
- await client.export_ovpack(
- uri="viking://agent/default/memories/",
- to="./exports/agent-memories.ovpack",
- )
- await client.import_ovpack(
- file_path="./exports/agent-memories.ovpack",
- parent="viking://agent/default/",
- on_conflict="overwrite",
- )
- finally:
- await client.close()
+```text
+type, context_type, level, name, description, tags, abstract
```
-### Export/Import Memories (HTTP API)
+These fields are regenerated in the target environment and are not restored
+from the package:
-```bash
-# Export user memories
-curl -X POST http://localhost:1933/api/v1/pack/export \
- -H "Content-Type: application/json" \
- -H "X-API-Key: your-key" \
- -d '{
- "uri": "viking://user/default/memories/",
- "to": "./exports/user-memories.ovpack"
- }'
-
-# Import user memories (upload first, then import via temp_file_id)
-TEMP_FILE_ID=$(
- curl -sS -X POST http://localhost:1933/api/v1/resources/temp_upload \
- -H "X-API-Key: your-key" \
- -F 'file=@./exports/user-memories.ovpack' \
- | jq -r '.result.temp_file_id'
-)
-curl -X POST http://localhost:1933/api/v1/pack/import \
- -H "Content-Type: application/json" \
- -H "X-API-Key: your-key" \
- -d "{
- \"temp_file_id\": \"$TEMP_FILE_ID\",
- \"parent\": \"viking://user/default/\",
- \"on_conflict\": \"overwrite\"
- }"
+```text
+id, uri, account_id, owner_user_id, owner_agent_id, owner_space,
+created_at, updated_at, active_count
```
-### Vectorization on Import
+With `--include-vectors`, export also stores pure-dense vectors and embedding
+metadata. Even when import restores dense snapshots, runtime fields are rebuilt
+from the target URI, target account, and current time. Hybrid indexes do not
+currently support vector snapshot export.
+
+## Import Validation
+
+Import validates the entire package before writing package content. Core checks:
+
+1. ZIP members must be under one package root and cannot contain absolute paths, backslashes, drive letters, or `..`.
+2. `<root>/_ovpack/manifest.json` must exist.
+3. `kind` must be `openviking.ovpack`, and `format_version` must equal the currently supported version.
+4. `root.name` must match the ZIP root, and the leaf of `root.uri` must also match `root.name`.
+5. The file set and directory set declared by the manifest must exactly match ZIP content.
+6. Each file `size` and `sha256` must match actual content.
+7. `content_sha256` must match the sorted file inventory.
+8. `_ovpack/index_records.jsonl` and optional `_ovpack/dense.f32` must match manifest hashes, counts, and dimensions.
+9. Source scope and target scope must match; structured scopes such as `user`, `agent`, and `session` also keep root depth stable.
+10. No package content is written before validation passes; conflict handling also runs before writes.
+
+Typical rejection examples:
+
+```text
+INVALID_ARGUMENT: Missing ovpack manifest
+INVALID_ARGUMENT: ovpack file sha256 does not match manifest
+INVALID_ARGUMENT: ovpack entries do not match manifest
+INVALID_ARGUMENT: ovpack source scope does not match target scope
+INVALID_ARGUMENT: ovpack package does not contain a dense vector snapshot
+```
-Imports always rebuild vectors in the target environment for `find/search`.
-OVPack no longer exposes an import option to disable vectorization. Session
-imports are restored as files only and are not vectorized.
+## Import Path Rules
-### Session Import and Export
+Regular subtree packages import into a parent directory in the same scope and
+keep the package root:
```bash
-openviking export viking://session/sess_123/ ./exports/sess_123.ovpack
-openviking import ./exports/sess_123.ovpack viking://session/ --on-conflict overwrite
+ov export viking://resources/a ./exports/a.ovpack
+ov import ./exports/a.ovpack viking://resources/imported/
```
-The restored root is `viking://session/sess_123/`.
+Result:
+
+```text
+viking://resources/imported/a
+```
-### Scope Root Import
+Top-level scope packages can only be imported to `viking://`:
```bash
-openviking export viking://resources/ ./backups/resources.ovpack
-openviking import ./backups/resources.ovpack viking:// --on-conflict overwrite
+ov export viking://resources ./exports/resources.ovpack
+ov import ./exports/resources.ovpack viking:// --on-conflict overwrite
```
-Scope-root packages cannot be imported under another scope directory.
+These imports are rejected:
+
+```bash
+# A resources package cannot be imported into session.
+ov import ./exports/a.ovpack viking://session/
+
+# A session subtree cannot be imported into resources.
+ov import ./exports/sess_123.ovpack viking://resources/
+
+# A session subtree cannot use itself as parent, which would create session/sess_123/sess_123.
+ov import ./exports/sess_123.ovpack viking://session/sess_123/
+```
-## Use Cases
+## Memories and Sessions
-### Full Backup and Migration
+Memory directories have fixed structures. Import the package into the matching
+parent directory to avoid duplicate path segments.
-Use a backup package for full migration instead of regular `export/import`
-parent-directory semantics:
+User memories:
```bash
-openviking backup ./backups/openviking.ovpack
-openviking restore ./backups/openviking.ovpack --on-conflict overwrite
+ov export viking://user/default/memories ./exports/user-memories.ovpack
+ov import ./exports/user-memories.ovpack viking://user/default/ --on-conflict overwrite
```
-Backup packages restore to the original public scope roots: `resources`, `user`,
-`agent`, and `session`. They exclude internal/runtime data such as `temp`,
-`queue`, lock files, OVPack manifests, and `.relations.json`. Non-session
-content is re-vectorized; session files are only restored.
+Agent memories:
-### Resource Backup
```bash
-DATE=$(date +%Y%m%d)
-openviking export viking://resources/ ./backups/backup_${DATE}.ovpack
+ov export viking://agent/default/memories ./exports/agent-memories.ovpack
+ov import ./exports/agent-memories.ovpack viking://agent/default/ --on-conflict overwrite
```
-### Resource Migration
-```bash
-# Export on Machine A
-openviking export viking://resources/my-project/ ./migration.ovpack
+Sessions restore file state only and do not trigger vectorization:
-# Import on Machine B
-openviking import ./migration.ovpack viking://resources/ --on-conflict overwrite
+```bash
+ov export viking://session/sess_123 ./exports/sess_123.ovpack
+ov import ./exports/sess_123.ovpack viking://session/ --on-conflict overwrite
```
-### Resource Sharing
-```bash
-# Export
-openviking export viking://resources/shared-docs/ ./shared-docs.ovpack
+Result:
-# Recipient imports
-openviking import ./shared-docs.ovpack viking://resources/team-shared/
+```text
+viking://session/sess_123
```
+## Old Packages and Future Versions
+
+The current implementation only accepts OVPack v2. Legacy packages without a
+manifest do not provide a file set, directory set, or checksums, so OpenViking
+cannot tell whether content was removed, modified, or mixed in. They are
+rejected by default. To migrate a legacy package, import it in a trusted old
+environment first, then re-export it with the current version.
+
+Future package versions are not silently accepted either. Upgrade OpenViking or
+re-export from an environment that can read that version.
+
+## Common Errors
+
+| Error | Common cause | Fix |
+| --- | --- | --- |
+| `Missing ovpack manifest` | Legacy package without a manifest | Re-export as v2 in a trusted environment. |
+| `Unsupported ovpack format_version` | Package format version is not currently supported | Upgrade OpenViking or re-export. |
+| `sha256 does not match manifest` | File or internal index content was changed | Discard the package or re-export from a trusted source. |
+| `ovpack entries do not match manifest` | ZIP content is missing files/directories or includes extra files/directories | Discard the package or re-export. |
+| `source scope does not match target scope` | Cross-scope import, such as session into resources | Import into a parent directory in the same scope. |
+| `source path is incompatible with target path` | Structured scope root depth would change | Import into the correct system parent directory. |
+| `Top-level scope ovpack packages must be imported to viking://` | A top-level scope package was imported to a non-root parent | Import to `viking://`. |
+| `Backup ovpack packages must be restored` | A backup package was imported with regular import | Use `ov restore`. |
+| `Resource already exists` | Target root already exists | Use `--on-conflict overwrite` or `--on-conflict skip`. |
+| `incomplete OpenViking vector index snapshot` | `--include-vectors` found missing index records in the export range | Run `ov system consistency <uri>` to locate the issue, then wait for processing or reindex. |
+| `dense vector snapshot is incompatible` | Package embedding metadata does not match current config | Use `--vector-mode recompute`, or switch to a compatible config. |
+
## FAQ
-**Q: Can I manually extract and view OVPack files?**
-A: Yes! OVPack is a standard ZIP format and can be opened with any compression tool.
+**Can I manually extract and inspect OVPack files?**
+
+Yes. OVPack is a ZIP file and can be opened with ordinary ZIP tools. Do not edit
+it manually before import, because edits break manifest validation. If both
+manifest and content are changed, use external signatures and trusted sources to
+decide whether the package is safe.
+
+**Why are vectors not exported by default?**
+
+Vectors are reusable only when the embedding model, input mode, parameters, and
+dimensions are fully compatible. Recomputing by default is safer. Use
+`--include-vectors` with `--vector-mode auto/require` when you need faster cold
+migration and know the environments are compatible.
-**Q: What if large OVPack imports are slow?**
-A: Imports now always rebuild vectors. If import time is too high, split the content into smaller OVPack files and import them in batches.
+**What if large package imports are slow?**
-**Q: How to handle duplicate resources during import?**
-A: Use `--on-conflict overwrite` to replace existing resources, or `--on-conflict skip` to keep them.
+Default import rebuilds target semantic and vector state. For large migrations,
+use `--include-vectors` to reduce recomputation, or split content into smaller
+OVPack files and import them in batches.
diff --git a/docs/zh/api/01-overview.md b/docs/zh/api/01-overview.md
index a59a1b942..148bc6d3e 100644
--- a/docs/zh/api/01-overview.md
+++ b/docs/zh/api/01-overview.md
@@ -348,6 +348,7 @@ JSON 输出 - 错误:
| GET | `/metrics` | Prometheus 指标导出 | 可选 |
| GET | `/api/v1/system/status` | 系统状态 | 需要 |
| POST | `/api/v1/system/wait` | 等待处理完成 | 需要 |
+| POST | `/api/v1/system/consistency` | 文件系统和向量索引一致性检查 | 需要 |
### 资源端点
diff --git a/docs/zh/api/03-filesystem.md b/docs/zh/api/03-filesystem.md
index cff8409c1..e3b9a969e 100644
--- a/docs/zh/api/03-filesystem.md
+++ b/docs/zh/api/03-filesystem.md
@@ -994,10 +994,12 @@ openviking unlink viking://resources/docs/auth/ viking://resources/docs/security
5. 以文件流形式返回
**格式说明**:
-- 导出的 ZIP 会包含 `<root>/_._ovpack_manifest.json`,这是 `.ovpack_manifest.json` 在 ZIP 内的转义名称。
+- 导出的 ZIP 会把用户内容原样放在 `<root>/files/` 下,并把内部元数据放在 `<root>/_ovpack/` 下。
+- manifest 位于 `<root>/_ovpack/manifest.json`。
- `entries[].path` 是相对导出 root 的路径;`""` 表示 root 目录本身。
- 文件条目包含 `size` 和 `sha256`;`content_sha256` 覆盖按路径排序后的文件列表(`path`、`size`、`sha256`)。
-- 原始 embedding 向量以及 `created_at`、`updated_at`、`active_count` 等运行态字段不会被导出。
+- `_ovpack/index_records.jsonl` 保存可迁移的索引标量。`include_vectors=true` 时,`_ovpack/dense.f32` 保存纯 dense float32 向量快照和 embedding 元数据;底层 `VectorIndex.IndexType` 为 hybrid 时不支持向量快照导出。
+- `id`、`uri`、`account_id`、`created_at`、`updated_at`、`active_count` 等运行态字段会在目标环境重新生成,不从包内恢复。
- OVPack 不额外设置包大小、文件数量或目录深度上限;实际可处理规模由 ZIP、存储后端和运行环境决定。
**代码入口**:
@@ -1012,6 +1014,7 @@ openviking unlink viking://resources/docs/auth/ viking://resources/docs/security
| 参数 | 类型 | 必填 | 默认值 | 说明 |
|------|------|------|--------|------|
| uri | string | 是 | - | 要导出的 Viking URI |
+| include_vectors | boolean | 否 | false | 导出纯 dense 向量快照;底层 index type 为 hybrid 时会拒绝 |
**权限要求**:ROOT 或 ADMIN
@@ -1029,7 +1032,8 @@ curl -X POST http://localhost:1933/api/v1/pack/export \
-H "Content-Type: application/json" \
-H "X-API-Key: your-admin-key" \
-d '{
- "uri": "viking://resources/my-project/"
+ "uri": "viking://resources/my-project/",
+ "include_vectors": false
}' \
--output my-project.ovpack
```
@@ -1051,6 +1055,9 @@ client.initialize()
```bash
# 导出资源
ov export viking://resources/my-project/ ./exports/my-project.ovpack
+
+# 导出 dense 向量快照
+ov export viking://resources/my-project/ ./exports/my-project.ovpack --include-vectors
```
**响应示例**
@@ -1088,11 +1095,14 @@ ov export viking://resources/my-project/ ./exports/my-project.ovpack
| temp_file_id | string | 是 | - | 临时上传文件 ID(通过 [temp_upload](02-resources.md#temp_upload) 获取) |
| parent | string | 是 | - | 目标父级 URI(导入到此处) |
| on_conflict | string | 否 | fail | 冲突策略:`fail`、`overwrite` 或 `skip` |
+| vector_mode | string | 否 | auto | 向量处理方式:`auto`、`recompute` 或 `require` |
**权限要求**:ROOT 或 ADMIN
**行为说明**:
-- 导入后总是在目标环境重建向量。API 已不再接受 `vectorize` 或 `force`。
+- API 已不再接受 `vectorize` 或 `force`。
+- `vector_mode=auto` 会在存在兼容 dense 快照时直接恢复,否则重新向量化;`recompute` 总是忽略包内向量;`require` 要求必须存在兼容 dense 快照,否则导入失败。
+- dense 快照兼容性会比较 embedding provider、model、input、query/document 参数和维度。
- Session 导入只恢复 session 文件状态,不触发向量化。
- `on_conflict=fail` 且目标 root 已存在时,会返回结构化的 `409 CONFLICT`。
- `on_conflict=overwrite` 会替换已有目标 root。`on_conflict=skip` 会保留已有目标 root,并直接返回该路径,不写入包内容。`skip` 是 root 级跳过,不是文件级补齐。
@@ -1100,7 +1110,7 @@ ov export viking://resources/my-project/ ./exports/my-project.ovpack
- 带 manifest entries 的包如果缺少内容文件或目录、混入额外文件或目录、文件大小不同、单文件 `sha256` 不同,或整体 `content_sha256` 缺失/不匹配,都会被拒绝导入。
- manifest `format_version` 不是当前支持版本的包会被拒绝。
- `.abstract.md` 和 `.overview.md` 会作为语义侧边文件恢复;`.relations.json` 和 OVPack 内部文件会被排除。
-- manifest `context_type` 必须和最终导入路径语义一致。
+- manifest index 标量中的 `context_type` 如果存在,必须和最终导入路径语义一致。
- `viking://resources/` 这类顶级 scope 包必须导入到 `viking://`。
- OVPack 不额外设置导入包大小、文件数量或目录深度上限;实际可处理规模由 ZIP、存储后端和运行环境决定。
@@ -1129,7 +1139,8 @@ curl -X POST http://localhost:1933/api/v1/pack/import \
-d "{
\"temp_file_id\": \"$TEMP_FILE_ID\",
\"parent\": \"viking://resources/imported/\",
- \"on_conflict\": \"overwrite\"
+ \"on_conflict\": \"overwrite\",
+ \"vector_mode\": \"auto\"
}"
```
@@ -1153,6 +1164,9 @@ ov import ./exports/my-project.ovpack viking://resources/imported/
# 显式冲突策略
ov import ./exports/my-project.ovpack viking://resources/imported/ --on-conflict overwrite
+
+# 要求恢复兼容 dense 向量快照
+ov import ./exports/my-project.ovpack viking://resources/imported/ --vector-mode require
```
**响应示例**
@@ -1190,6 +1204,7 @@ ov import ./exports/my-project.ovpack viking://resources/imported/ --on-conflict
将公开 scope root 备份为只能通过 restore 恢复的 `.ovpack` 文件。备份包含
`resources`、`user`、`agent`、`session`,不包含 `temp`、`queue` 等内部运行态数据。
+设置 `include_vectors=true` 时,会额外导出兼容的纯 dense 向量快照;底层 index type 为 hybrid 时会拒绝导出向量快照。
```
POST /api/v1/pack/backup
@@ -1197,7 +1212,9 @@ POST /api/v1/pack/backup
```bash
curl -X POST http://localhost:1933/api/v1/pack/backup \
+ -H "Content-Type: application/json" \
-H "X-API-Key: your-admin-key" \
+ -d '{"include_vectors":false}' \
--output openviking-backup.ovpack
```
@@ -1205,6 +1222,7 @@ CLI:
```bash
ov backup ./backups/openviking.ovpack
+ov backup ./backups/openviking.ovpack --include-vectors
```
---
@@ -1212,12 +1230,13 @@ ov backup ./backups/openviking.ovpack
### restore_ovpack
恢复 `backup_ovpack` 生成的备份包到原始公开 scope root。普通 import 不接受备份包。
-非 session 内容恢复后重新向量化;session 只恢复文件状态。
+向量处理遵循 `vector_mode`;session 只恢复文件状态。
| 参数 | 类型 | 必填 | 默认值 | 说明 |
|------|------|------|--------|------|
| temp_file_id | string | 是 | - | 临时上传文件 ID |
| on_conflict | string | 否 | fail | 冲突策略:`fail`、`overwrite` 或 `skip` |
+| vector_mode | string | 否 | auto | 向量处理方式:`auto`、`recompute` 或 `require` |
```
POST /api/v1/pack/restore
@@ -1235,13 +1254,14 @@ TEMP_FILE_ID=$(
curl -X POST http://localhost:1933/api/v1/pack/restore \
-H "Content-Type: application/json" \
-H "X-API-Key: your-admin-key" \
- -d "{\"temp_file_id\":\"$TEMP_FILE_ID\",\"on_conflict\":\"overwrite\"}"
+ -d "{\"temp_file_id\":\"$TEMP_FILE_ID\",\"on_conflict\":\"overwrite\",\"vector_mode\":\"auto\"}"
```
CLI:
```bash
ov restore ./backups/openviking.ovpack --on-conflict overwrite
+ov restore ./backups/openviking.ovpack --on-conflict overwrite --vector-mode require
```
---
diff --git a/docs/zh/api/07-system.md b/docs/zh/api/07-system.md
index 0c5de4c10..d17b3dbf5 100644
--- a/docs/zh/api/07-system.md
+++ b/docs/zh/api/07-system.md
@@ -165,6 +165,79 @@ ov system status
---
+### consistency
+
+#### 1. API 实现介绍
+
+检查指定 URI 子树的文件系统内容和向量索引是否一致,用于调试索引缺失、向量快照导出失败等问题。该能力是通用数据一致性检查,不属于 OVPack 私有接口;`ov export --include-vectors` 和 `ov backup --include-vectors` 会复用同一检查。
+
+响应只返回摘要和缺失项,不返回完整 expected 列表。`missing_records` 最多返回前 20 条;如果还有更多缺失项,`missing_records_truncated` 为 `true`。
+
+**代码入口**:
+- `openviking/server/routers/system.py:check_consistency` - HTTP 路由
+- `openviking_cli/client/sync_http.py:SyncHTTPClient.check_consistency` - SDK 入口
+- `crates/ov_cli/src/commands/system.rs:consistency` - CLI 命令
+
+#### 2. 接口和参数说明
+
+| 参数 | 类型 | 必填 | 默认值 | 说明 |
+|------|------|------|--------|------|
+| uri | string | 是 | - | 要检查的 Viking URI 子树 |
+
+#### 3. 使用示例
+
+**HTTP API**
+
+```
+POST /api/v1/system/consistency
+Content-Type: application/json
+```
+
+```bash
+curl -X POST http://localhost:1933/api/v1/system/consistency \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-key" \
+ -d '{"uri":"viking://resources/my-project"}'
+```
+
+**Python SDK**
+
+```python
+report = client.check_consistency("viking://resources/my-project")
+print(report["ok"])
+print(report["missing_records"])
+```
+
+**CLI**
+
+```bash
+ov system consistency viking://resources/my-project
+```
+
+**响应示例**
+
+```json
+{
+ "status": "ok",
+ "result": {
+ "ok": false,
+ "expected_count": 3,
+ "missing_record_count": 1,
+ "missing_records_truncated": false,
+ "missing_records": [
+ {
+ "uri": "viking://resources/my-project/README.md",
+ "path": "README.md",
+ "level": 2,
+ "key": "README.md#level=2"
+ }
+ ]
+ }
+}
+```
+
+---
+
### wait_processed
#### 1. API 实现介绍
diff --git a/docs/zh/guides/09-ovpack.md b/docs/zh/guides/09-ovpack.md
index f516e9311..b9ca3031f 100644
--- a/docs/zh/guides/09-ovpack.md
+++ b/docs/zh/guides/09-ovpack.md
@@ -1,100 +1,176 @@
# OVPack 导入导出
-OVPack 是 OpenViking 的上下文打包格式,用来把一个 `viking://` 子树导出成
-`.ovpack` 文件,再导入到另一个 OpenViking 环境。它适合备份、迁移、分享资源或记忆
-目录;它不是长期归档格式,也不是带签名的安全发布格式。
+OVPack 是 OpenViking 的可恢复内容包格式,用来迁移或备份 `viking://` 下的公开内容树。
+它保存文件内容、语义侧边文件、可迁移的索引标量,以及可选的 dense 向量快照。
-## 适用边界
+OVPack 不是裸 ZIP 拷贝,也不是可信发布格式。导入会校验 manifest、文件列表、目录列表和
+checksum,保证包内容没有偏离 manifest;如果攻击者能同时篡改文件和 manifest,仍需要依赖
+外部签名、传输安全和访问控制。
-OVPack 可以处理这些公开作用域:
+## 支持范围
+
+普通 `export/import` 处理一个包根:
- `viking://resources/...`
- `viking://user/...`
- `viking://agent/...`
- `viking://session/...`
-OVPack 支持导出公开 scope root,例如 `viking://resources/`、`viking://user/`、
-`viking://agent/`、`viking://session/`;这类顶级 scope 包只能导入到 `viking://`。
-全量备份使用单独的 `backup` / `restore` 接口,自动包含这些公开 scope root。
-`temp`、`queue` 等内部作用域不属于 OVPack 迁移范围。
+全量迁移使用单独的 `backup/restore`,它会把公开 scope root 一起打进备份包:
+
+- `viking://resources`
+- `viking://user`
+- `viking://agent`
+- `viking://session`
-`.abstract.md` 和 `.overview.md` 会作为语义侧边文件随包恢复;`.relations.json`、锁文件、
-manifest 等内部文件不会进入包。导入后 OpenViking 会在目标环境重新生成语义和向量;session
-包只恢复 session 文件状态,不触发向量化。导入接口不再提供 `vectorize` 或 `force` 参数;
-冲突处理统一使用 `on_conflict`。OVPack 本身不再设置额外的包大小、文件数量或目录深度上限;
-实际可处理规模由 ZIP、存储后端和运行环境决定。
+`temp`、`queue`、`upload`、锁文件、watch control 文件、`.relations.json` 等内部或运行态数据
+不属于 OVPack 迁移范围。
## 快速开始
-### CLI
+### 导出和导入资源目录
```bash
-# 导出资源目录
-ov export viking://resources/my-project/ ./exports/my-project.ovpack
-
-# 导入到目标父目录
+ov export viking://resources/my-project ./exports/my-project.ovpack
ov import ./exports/my-project.ovpack viking://resources/imported/
+```
-# 目标 root 已存在时覆盖
+导入参数是目标父目录,不是最终 root。假设包根名是 `my-project`,上面的导入结果是:
+
+```text
+viking://resources/imported/my-project
+```
+
+覆盖已有 root:
+
+```bash
ov import ./exports/my-project.ovpack viking://resources/imported/ --on-conflict overwrite
```
-导入的第二个参数是“父目录”,不是最终 root。假设包内 root 名是 `my-project`:
+### 导出向量快照
-```text
+默认导出不保存 dense 向量,导入后由目标环境重新向量化:
+
+```bash
+ov export viking://resources/my-project ./exports/my-project.ovpack
ov import ./exports/my-project.ovpack viking://resources/imported/
```
-导入结果是:
+如果确认导出环境和导入环境使用同一 embedding 配置,可以显式导出 dense 向量快照:
-```text
-viking://resources/imported/my-project
+```bash
+ov export viking://resources/my-project ./exports/my-project.ovpack --include-vectors
+ov import ./exports/my-project.ovpack viking://resources/imported/ --vector-mode auto
+```
+
+`--vector-mode` 控制导入时如何处理包内向量:
+
+| 值 | 行为 |
+| --- | --- |
+| `auto` | 默认值。若包内有 dense 快照且 embedding 元数据兼容,则直接恢复;否则重新向量化。 |
+| `recompute` | 忽略包内 dense 快照,始终重新向量化。 |
+| `require` | 必须恢复兼容 dense 快照;没有快照、快照不完整、模型或维度不兼容都会报错。 |
+
+兼容性校验会比较包内记录的 embedding provider、model、input、query/document 参数和维度。
+当前 OVPack 向量快照只支持纯 dense 索引;如果底层向量索引的 `VectorIndex.IndexType` 是 hybrid,`--include-vectors` 会直接拒绝导出。导入到 hybrid index 环境时,`auto` 会重新向量化,`require` 会报错。
+
+导出 dense 向量快照前,OpenViking 会先做数据一致性检查。也就是检查导出范围内按系统规则
+应该进入向量索引的内容,是否已经有对应索引记录。缺失时会拒绝导出,避免生成不完整的
+迁移包。
+
+可以单独调用一致性检查来调试当前数据状态:
+
+```bash
+ov system consistency viking://resources/my-project
+```
+
+接口只返回摘要和最多 20 条缺失记录,不返回完整 expected 列表。`--include-vectors`
+导出失败时,错误 details 只携带 1 条缺失 key,避免错误日志过大。
+
+Python SDK:
+
+```python
+report = await client.check_consistency("viking://resources/my-project")
+print(report["ok"], report["missing_records"])
+```
+
+HTTP API:
+
+```bash
+curl -X POST http://localhost:1933/api/v1/system/consistency \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-admin-key" \
+ -d '{"uri":"viking://resources/my-project"}'
+```
+
+### 全量备份和恢复
+
+全量迁移不要用 `export viking://`;使用专门的备份包:
+
+```bash
+ov backup ./backups/openviking.ovpack
+ov restore ./backups/openviking.ovpack --on-conflict overwrite
```
-### Python SDK
+备份包只能通过 `restore` 恢复,不能通过普通 `import` 导入到任意父目录。
+
+## Python SDK
```python
from openviking import AsyncOpenViking
-async def export_and_import():
+async def migrate_project():
client = AsyncOpenViking()
await client.initialize()
try:
await client.export_ovpack(
- uri="viking://resources/my-project/",
+ uri="viking://resources/my-project",
to="./exports/my-project.ovpack",
+ include_vectors=False,
)
imported_uri = await client.import_ovpack(
file_path="./exports/my-project.ovpack",
parent="viking://resources/imported/",
on_conflict="overwrite",
+ vector_mode="auto",
)
print(imported_uri)
-
- # 导入会触发目标环境重新向量化;需要检索前可等待后台任务完成。
await client.wait_processed()
finally:
await client.close()
```
-### HTTP API
+全量备份:
+
+```python
+await client.backup_ovpack("./backups/openviking.ovpack", include_vectors=True)
+await client.restore_ovpack(
+ "./backups/openviking.ovpack",
+ on_conflict="overwrite",
+ vector_mode="auto",
+)
+```
+
+## HTTP API
+
+HTTP 导出接口直接返回文件流;HTTP 导入和恢复必须先上传本地 `.ovpack`,再用
+`temp_file_id` 调用 pack 接口。
-HTTP 导出接口直接返回文件流;HTTP 导入必须先上传本地 `.ovpack` 文件,再用
-`temp_file_id` 导入。
+导出:
```bash
-# 导出
curl -X POST http://localhost:1933/api/v1/pack/export \
-H "Content-Type: application/json" \
-H "X-API-Key: your-admin-key" \
- -d '{"uri": "viking://resources/my-project/"}' \
+ -d '{"uri":"viking://resources/my-project","include_vectors":false}' \
--output my-project.ovpack
```
+导入:
+
```bash
-# 第一步:上传本地 ovpack 文件
TEMP_FILE_ID=$(
curl -sS -X POST http://localhost:1933/api/v1/resources/temp_upload \
-H "X-API-Key: your-admin-key" \
@@ -102,19 +178,26 @@ TEMP_FILE_ID=$(
| jq -r ".result.temp_file_id"
)
-# 第二步:导入
curl -X POST http://localhost:1933/api/v1/pack/import \
-H "Content-Type: application/json" \
-H "X-API-Key: your-admin-key" \
-d "{
\"temp_file_id\": \"$TEMP_FILE_ID\",
\"parent\": \"viking://resources/imported/\",
- \"on_conflict\": \"overwrite\"
+ \"on_conflict\": \"overwrite\",
+ \"vector_mode\": \"auto\"
}"
```
-HTTP 接口不接受本机路径字段,例如 `file_path`、`temp_path`。本地文件读取和上传由 CLI 或
-SDK 客户端完成,服务端只消费 `temp_file_id`。
+全量备份:
+
+```bash
+curl -X POST http://localhost:1933/api/v1/pack/backup \
+ -H "Content-Type: application/json" \
+ -H "X-API-Key: your-admin-key" \
+ -d '{"include_vectors":true}' \
+ --output openviking-backup.ovpack
+```
## 冲突策略
@@ -123,41 +206,31 @@ SDK 客户端完成,服务端只消费 `temp_file_id`。
| 值 | 行为 |
| --- | --- |
| `fail` | 默认值。目标 root 已存在时返回 `409 CONFLICT`。 |
-| `overwrite` | 删除已有 root,再写入包内容。 |
-| `skip` | 保留已有 root,直接返回该 URI,不写入包内容。 |
-
-`skip` 是 root 级跳过,不是文件级补齐导入:只要目标 root 已存在,就不会写入包内缺失文件。
+| `overwrite` | 删除已有 root,再写入包内容和重建索引。 |
+| `skip` | 目标 root 已存在时直接返回该 URI,不写入任何包内容。 |
-示例错误:
-
-```json
-{
- "status": "error",
- "error": {
- "code": "CONFLICT",
- "message": "Resource already exists at viking://resources/imported/my-project. Use on_conflict='overwrite' to replace it.",
- "details": {
- "resource": "viking://resources/imported/my-project"
- }
- }
-}
-```
+`skip` 是 root 级跳过,不是文件级补齐导入。
## 包结构
-OVPack v2 是标准 ZIP 文件。ZIP 内部以一个 root 目录包住所有内容,并包含一个 manifest:
+OVPack v2 是标准 ZIP。ZIP 内部有一个包根目录:
```text
my-project/
-my-project/notes.txt
-my-project/_._overview.md
-my-project/_._ovpack_manifest.json
+my-project/files/
+my-project/files/notes.txt
+my-project/files/.abstract.md
+my-project/files/.overview.md
+my-project/_ovpack/
+my-project/_ovpack/index_records.jsonl
+my-project/_ovpack/dense.f32 # 仅 --include-vectors 且存在可导出向量时出现
+my-project/_ovpack/manifest.json
```
-`_._ovpack_manifest.json` 是 `.ovpack_manifest.json` 在 ZIP 内的转义名称。OpenViking 会把
-普通内容里的点文件名也按同一规则转义,例如 `.overview.md` 在 ZIP 内是 `_._overview.md`。
+`files/` 下保存用户内容,路径与 OpenViking 中的相对路径完全一致,不再对点文件做 `_._` 转义。
+`_ovpack/` 下保存 OVPack 内部文件,不参与用户内容导入。
-一个最小 manifest 示例:
+manifest 只保存包结构、文件 checksum 和内部索引文件的 checksum,不直接内嵌每个文件的索引记录:
```json
{
@@ -169,10 +242,7 @@ my-project/_._ovpack_manifest.json
"scope": "resources"
},
"entries": [
- {
- "path": "",
- "kind": "directory"
- },
+ {"path": "", "kind": "directory"},
{
"path": "notes.txt",
"kind": "file",
@@ -181,214 +251,199 @@ my-project/_._ovpack_manifest.json
}
],
"content_sha256": "b2a6e9582119c7510d68e3446de3e71a486934bf450d68f65596259ed1cf7997",
- "vectors": {
- "": [
- {
- "level": 0,
- "text": "Project summary",
- "scalars": {
- "context_type": "resource",
- "level": 0,
- "abstract": "Project summary"
- }
+ "index": {
+ "records": {
+ "path": "_ovpack/index_records.jsonl",
+ "count": 2,
+ "sha256": "..."
+ },
+ "dense": {
+ "path": "_ovpack/dense.f32",
+ "count": 1,
+ "dtype": "float32",
+ "byte_order": "little",
+ "dimensions": 1024,
+ "sha256": "...",
+ "embedding": {
+ "provider": "volcengine",
+ "model": "doubao-embedding-vision-251215",
+ "input": "multimodal",
+ "dimensions": 1024
}
- ]
+ }
}
}
```
-字段说明:
-
-- `kind` 固定为 `openviking.ovpack`。
-- `format_version` 当前为 `2`。低于或高于当前支持版本的包都会被拒绝。
-- `root.name` 是导入时创建的 root 名称。
-- `entries[].path` 是相对 root 的路径。`""` 表示 root 目录本身。
-- 文件条目包含 `size` 和 `sha256`。
-- `content_sha256` 覆盖按路径排序后的文件列表,元素只包含 `path`、`size`、`sha256`。
-- `vectors` 只保存可迁移的标量和目录摘要文本,不保存原始 embedding 向量。
-- 包内如果带有 `context_type`,只作为导出时的标量信息;导入校验不依赖它,最终
- `context_type` 以目标路径推导结果为准。
+`entries[].path == ""` 表示包根目录本身。多层目录使用相对路径,例如:
-当前会导出的向量标量字段是:
-
-```text
-type, context_type, level, name, description, tags, abstract
+```json
+[
+ {"path": "", "kind": "directory"},
+ {"path": "docs", "kind": "directory"},
+ {"path": "docs/a.md", "kind": "file", "size": 12, "sha256": "..."}
+]
```
-这些运行态字段不会导出,会在目标环境重新生成:
+`index_records.jsonl` 一行一条索引记录,覆盖文件、目录、根目录:
-```text
-created_at, updated_at, active_count
+```jsonl
+{"record_id":"r000001","path":"","kind":"directory","level":0,"text":"root abstract","scalars":{"abstract":"root abstract","context_type":"resource","level":0}}
+{"record_id":"r000002","path":"notes.txt","kind":"file","level":2,"scalars":{"abstract":"note summary","tags":"demo"},"vector":{"dense":{"offset":0,"dimensions":1024}}}
```
-## 导入校验
-
-导入会先校验包,再写入目标目录。核心规则如下:
+`dense.f32` 是连续 little-endian float32 数组。`index_records.jsonl` 中的 `vector.dense.offset`
+是 float 数组偏移,不是字节偏移。示例中 `offset=0, dimensions=1024` 表示从第 0 个 float
+开始读取 1024 个 float。
-1. ZIP 路径必须在同一个 root 下,不能包含绝对路径、反斜杠、盘符或 `..`。
-2. 必须存在 `<root>/_._ovpack_manifest.json`。
-3. manifest 必须是合法 JSON,`kind` 和 `format_version` 必须可识别。
-4. `entries` 中声明的文件和目录集合必须和 ZIP 内容一致。
-5. 每个文件的 `size` 和 `sha256` 必须匹配实际内容。
-6. v2 包必须带 `content_sha256`,并且整体 checksum 必须匹配。
-7. manifest root 的 source scope 必须和最终导入 root 的 target scope 一致。
-8. `user`、`agent`、`session` 这类结构化 scope 不能通过导入改变 root 层级。
-9. `.relations.json`、manifest、锁文件等内部文件不会作为普通内容导入。
+## 索引字段
-如果旧包没有 manifest,会被拒绝:
+默认会导出可迁移的索引标量:
```text
-INVALID_ARGUMENT: Missing ovpack manifest
+type, context_type, level, name, description, tags, abstract
```
-如果内容被改动,例如 manifest 里声明 `notes.txt` 的 sha256 是 `hello`,但 ZIP 里实际内容
-变成 `jello`,会被拒绝:
+这些字段不会从包里直接恢复,而是在目标环境重新生成:
```text
-INVALID_ARGUMENT: ovpack file sha256 does not match manifest
+id, uri, account_id, owner_user_id, owner_agent_id, owner_space,
+created_at, updated_at, active_count
```
-这个校验保证的是“包内容没有偏离 manifest”。如果攻击者能同时改 manifest 和文件内容,
-它不能替代签名、可信发布链或访问控制。
-
-如果包从 `viking://session/sess_123` 导出,却导入到 `viking://resources/`,会因为
-source scope 和 target scope 不一致被拒绝。导入到 `viking://session/sess_123/` 也会
-被拒绝,因为它会生成 `viking://session/sess_123/sess_123` 这种嵌套 root。
+如果使用 `--include-vectors`,会额外导出纯 dense 向量和 embedding 元数据。导入时即使恢复
+dense 快照,也会按目标 URI、目标账号和当前时间重建运行态字段。hybrid index 当前不支持
+向量快照导出。
-## 旧包和未来版本
+## 导入校验
-默认导入只接受带 manifest 的 OVPack。旧版无 manifest 包没有文件集合和 checksum 信息,
-无法判断是否混入或删改内容,因此默认拒绝。需要迁移旧包时,应先在可信环境中用旧版本导入,
-再用当前版本重新导出。
+导入会先完整校验,再写入目标环境。核心校验包括:
-开发期间生成过的 v2 预览包如果缺少 `content_sha256`,也会被拒绝。处理方式同样是重新导出。
+1. ZIP 成员必须都在同一个包根下,不能包含绝对路径、反斜杠、盘符或 `..`。
+2. 必须存在 `<root>/_ovpack/manifest.json`。
+3. `kind` 必须是 `openviking.ovpack`,`format_version` 必须等于当前支持版本。
+4. `root.name` 必须和 ZIP 根目录一致,`root.uri` 的末段也必须和 `root.name` 一致。
+5. manifest 声明的文件集合、目录集合必须和 ZIP 内容一致,不能缺失也不能混入额外内容。
+6. 每个文件的 `size` 和 `sha256` 必须匹配实际内容。
+7. `content_sha256` 必须匹配按路径排序后的文件清单。
+8. `_ovpack/index_records.jsonl` 和可选 `_ovpack/dense.f32` 必须匹配 manifest 中的 hash、数量和维度。
+9. source scope 和 target scope 必须一致;`user`、`agent`、`session` 这类结构化 scope 还要求 root 层级一致。
+10. 校验通过前不会写入包内容;冲突策略也在写入前处理。
-如果 manifest 的 `format_version` 不是当前支持版本,会返回不支持的版本错误。低版本不会自动
-兼容;处理方式是在可信环境中重新导出为当前支持格式。未来版本包则应升级 OpenViking,或由
-支持该版本的环境重新导出成当前支持的格式。
+典型拒绝示例:
-## 记忆迁移
+```text
+INVALID_ARGUMENT: Missing ovpack manifest
+INVALID_ARGUMENT: ovpack file sha256 does not match manifest
+INVALID_ARGUMENT: ovpack entries do not match manifest
+INVALID_ARGUMENT: ovpack source scope does not match target scope
+INVALID_ARGUMENT: ovpack package does not contain a dense vector snapshot
+```
-记忆目录有固定结构。导入时要把包导入到“对应目录的父目录”,避免产生重复路径。
+## 导入路径规则
-### 用户记忆
+普通子树包导入到同 scope 的父目录,并保留包根:
```bash
-# 导出整个用户 memories 子树
-ov export viking://user/default/memories/ ./exports/user-memories.ovpack
-
-# 导入到 user space 根目录,结果是 viking://user/default/memories/
-ov import ./exports/user-memories.ovpack viking://user/default/ --on-conflict overwrite
+ov export viking://resources/a ./exports/a.ovpack
+ov import ./exports/a.ovpack viking://resources/imported/
```
-不要导入到 `viking://user/default/memories/`,否则会得到:
+结果:
```text
-viking://user/default/memories/memories
+viking://resources/imported/a
```
-### Agent 记忆
+顶级 scope 包只能导入到 `viking://`:
```bash
-# isolate_agent_scope_by_user = false
-ov export viking://agent/default/memories/ ./exports/agent-memories.ovpack
-ov import ./exports/agent-memories.ovpack viking://agent/default/ --on-conflict overwrite
-
-# isolate_agent_scope_by_user = true
-ov export viking://agent/default/user/alice/memories/ ./exports/agent-memories.ovpack
-ov import ./exports/agent-memories.ovpack viking://agent/default/user/alice/ --on-conflict overwrite
+ov export viking://resources ./exports/resources.ovpack
+ov import ./exports/resources.ovpack viking:// --on-conflict overwrite
```
-## Session 迁移
-
-Session 包按备份恢复语义处理:恢复原 session id,不进行向量化。
+以下导入会被拒绝:
```bash
-ov export viking://session/sess_123/ ./exports/sess_123.ovpack
-ov import ./exports/sess_123.ovpack viking://session/ --on-conflict overwrite
-```
+# resources 包不能导入 session
+ov import ./exports/a.ovpack viking://session/
-导入结果是:
+# session 子树不能导入 resources
+ov import ./exports/sess_123.ovpack viking://resources/
-```text
-viking://session/sess_123/
+# session 子树不能把自身路径当父目录,否则会变成 session/sess_123/sess_123
+ov import ./exports/sess_123.ovpack viking://session/sess_123/
```
-如果导出整个 session scope:
-
-```bash
-ov export viking://session/ ./exports/session.ovpack
-ov import ./exports/session.ovpack viking:// --on-conflict overwrite
-```
+## 记忆和 Session
-## 常见场景
+记忆目录有固定结构。导入时把包导入到对应目录的父目录,避免产生重复路径。
-### 全量备份和迁移
-
-全量迁移使用专门的备份包,而不是普通 `export/import` 的父目录语义:
+用户记忆:
```bash
-ov backup ./backups/openviking.ovpack
-ov restore ./backups/openviking.ovpack --on-conflict overwrite
+ov export viking://user/default/memories ./exports/user-memories.ovpack
+ov import ./exports/user-memories.ovpack viking://user/default/ --on-conflict overwrite
```
-备份包恢复到原始公开 scope root:`resources`、`user`、`agent`、`session`。
-它不包含 `temp`、`queue`、锁文件、manifest、`.relations.json` 等内部或运行态数据。
-非 session 内容恢复后重新向量化;session 只恢复文件状态。
-
-### 备份资源
+Agent 记忆:
```bash
-DATE=$(date +%Y%m%d)
-ov export viking://resources/ ./backups/resources_${DATE}.ovpack
-
-# 恢复整个 resources scope 时,父目录必须是 viking://
-ov import ./backups/resources_${DATE}.ovpack viking:// --on-conflict overwrite
+ov export viking://agent/default/memories ./exports/agent-memories.ovpack
+ov import ./exports/agent-memories.ovpack viking://agent/default/ --on-conflict overwrite
```
-### 跨环境迁移
+Session 只恢复文件状态,不触发向量化:
```bash
-# 机器 A
-ov export viking://resources/my-project/ ./migration.ovpack
-
-# 机器 B
-ov import ./migration.ovpack viking://resources/ --on-conflict overwrite
+ov export viking://session/sess_123 ./exports/sess_123.ovpack
+ov import ./exports/sess_123.ovpack viking://session/ --on-conflict overwrite
```
-### 分享一组资源
+结果:
-```bash
-ov export viking://resources/shared-docs/ ./shared-docs.ovpack
-ov import ./shared-docs.ovpack viking://resources/team-shared/
+```text
+viking://session/sess_123
```
+## 旧包和未来版本
+
+当前实现只接受 OVPack v2。旧版无 manifest 包没有文件集合、目录集合和 checksum 信息,无法判断
+是否被删改或混入内容,因此默认拒绝。需要迁移旧包时,应先在可信旧环境中导入,再用当前版本
+重新导出。
+
+未来版本包也不会静默兼容。处理方式是升级 OpenViking,或在支持该版本的环境中重新导出为当前
+支持格式。
+
## 常见错误
| 错误 | 常见原因 | 处理方式 |
| --- | --- | --- |
-| `Missing ovpack manifest` | 旧版无 manifest 包 | 在可信环境重新导出。 |
-| `Missing ovpack manifest content_sha256` | 开发期 v2 预览包缺少整体 checksum | 重新导出。 |
-| `sha256 does not match manifest` | 文件内容和 manifest 不一致 | 丢弃该包,或从可信源重新导出。 |
+| `Missing ovpack manifest` | 旧版无 manifest 包 | 在可信环境重新导出为 v2。 |
+| `Unsupported ovpack format_version` | 包格式版本不是当前支持版本 | 升级 OpenViking 或重新导出。 |
+| `sha256 does not match manifest` | 文件或内部索引内容被改动 | 丢弃该包,或从可信源重新导出。 |
| `ovpack entries do not match manifest` | ZIP 中缺文件/目录,或混入额外文件/目录 | 丢弃该包,或重新导出。 |
-| `source scope does not match target scope` | 将包导入到了不同 scope,例如 session 导入 resources | 导入到同 scope 的父目录。 |
-| `source path is incompatible with target path` | 结构化 scope 的导入会改变 root 层级,例如 session 导入具体 session 内部 | 导入到正确的系统父目录。 |
-| `Top-level scope ovpack packages must be imported to viking://` | 将 `resources`/`user`/`agent`/`session` 顶级包导入了非根父目录 | 改为导入 `viking://`。 |
-| `Unsupported ovpack format_version` | 包格式版本不是当前支持版本 | 升级 OpenViking 或重新导出为当前支持版本。 |
+| `source scope does not match target scope` | 跨 scope 导入,例如 session 导入 resources | 导入到同 scope 的父目录。 |
+| `source path is incompatible with target path` | 结构化 scope 的 root 层级会改变 | 导入到正确系统父目录。 |
+| `Top-level scope ovpack packages must be imported to viking://` | 将顶级 scope 包导入了非根父目录 | 改为导入 `viking://`。 |
+| `Backup ovpack packages must be restored` | 用普通 import 导入 backup 包 | 使用 `ov restore`。 |
| `Resource already exists` | 目标 root 已存在 | 使用 `--on-conflict overwrite` 或 `--on-conflict skip`。 |
+| `incomplete OpenViking vector index snapshot` | 使用 `--include-vectors` 时,导出范围内应索引内容缺少索引记录 | 先执行 `ov system consistency <uri>` 定位问题,再等待处理完成或重新 reindex。 |
+| `dense vector snapshot is incompatible` | 包内 embedding 元数据和当前配置不一致 | 用 `--vector-mode recompute`,或换到兼容配置。 |
## 常见问题
**OVPack 可以手动解压查看吗?**
-可以。OVPack 是 ZIP 文件,可以用普通解压工具查看。不要手动改包后再导入;修改内容会破坏
-manifest 校验。
+可以。OVPack 是 ZIP 文件,可以用普通解压工具查看。不要手动修改后再导入,修改会破坏
+manifest 校验;如果同时修改 manifest 和内容,则需要依赖外部签名和可信来源判断。
-**大包导入很慢怎么办?**
+**为什么不默认导出向量?**
-导入会重建目标环境的语义和向量。大包迁移建议按目录拆成多个 OVPack 分批导入。
+向量只在 embedding 模型、输入模式、参数和维度完全兼容时才可直接复用。默认重新向量化更稳;
+需要冷迁移加速时,再显式使用 `--include-vectors` 和 `--vector-mode auto/require`。
-**为什么不兼容旧版无 manifest 包?**
+**大包导入很慢怎么办?**
-旧包没有可验证的文件列表,也没有 checksum。默认兼容会让被删改或混入文件的包直接写入目标
-环境,风险高于收益。
+默认导入会重建目标环境的语义和向量。大包迁移可以使用 `--include-vectors` 减少重算,或按目录
+拆成多个 OVPack 分批导入。
diff --git a/openviking/async_client.py b/openviking/async_client.py
index 61dd4436e..d0bf84b22 100644
--- a/openviking/async_client.py
+++ b/openviking/async_client.py
@@ -549,7 +549,12 @@ class AsyncOpenViking:
# ============= Pack methods =============
- async def export_ovpack(self, uri: str, to: str) -> str:
+ async def export_ovpack(
+ self,
+ uri: str,
+ to: str,
+ include_vectors: bool = False,
+ ) -> str:
"""
Export specified context path as .ovpack file.
@@ -561,9 +566,13 @@ class AsyncOpenViking:
Exported file path
"""
await self._ensure_initialized()
- return await self._client.export_ovpack(uri, to)
+ return await self._client.export_ovpack(
+ uri,
+ to,
+ include_vectors=include_vectors,
+ )
- async def backup_ovpack(self, to: str) -> str:
+ async def backup_ovpack(self, to: str, include_vectors: bool = False) -> str:
"""
Back up public OpenViking scopes as a restore-only .ovpack file.
@@ -574,13 +583,14 @@ class AsyncOpenViking:
Exported backup file path
"""
await self._ensure_initialized()
- return await self._client.backup_ovpack(to)
+ return await self._client.backup_ovpack(to, include_vectors=include_vectors)
async def import_ovpack(
self,
file_path: str,
parent: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""
Import local .ovpack file to specified parent path.
@@ -589,6 +599,7 @@ class AsyncOpenViking:
file_path: Local .ovpack file path
parent: Target parent URI (e.g., viking://user/alice/resources/references/)
on_conflict: One of "fail", "overwrite", or "skip"
+ vector_mode: One of "auto", "recompute", or "require"
Returns:
Imported root resource URI
@@ -598,12 +609,14 @@ class AsyncOpenViking:
file_path,
parent,
on_conflict=on_conflict,
+ vector_mode=vector_mode,
)
async def restore_ovpack(
self,
file_path: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""
Restore a backup .ovpack file to its original public scope roots.
@@ -611,6 +624,7 @@ class AsyncOpenViking:
Args:
file_path: Local backup .ovpack file path
on_conflict: One of "fail", "overwrite", or "skip"
+ vector_mode: One of "auto", "recompute", or "require"
Returns:
Restored root URI
@@ -619,10 +633,16 @@ class AsyncOpenViking:
return await self._client.restore_ovpack(
file_path,
on_conflict=on_conflict,
+ vector_mode=vector_mode,
)
# ============= Debug methods =============
+ async def check_consistency(self, uri: str) -> Dict[str, Any]:
+ """Check filesystem/vector-index consistency for a URI subtree."""
+ await self._ensure_initialized()
+ return await self._client.check_consistency(uri)
+
def get_status(self) -> Union[SystemStatus, Dict[str, Any]]:
"""Get system status.
diff --git a/openviking/client/local.py b/openviking/client/local.py
index 9f1e1eac9..f4ee93319 100644
--- a/openviking/client/local.py
+++ b/openviking/client/local.py
@@ -549,19 +549,34 @@ class LocalClient(BaseClient):
# ============= Pack =============
- async def export_ovpack(self, uri: str, to: str) -> str:
+ async def export_ovpack(
+ self,
+ uri: str,
+ to: str,
+ include_vectors: bool = False,
+ ) -> str:
"""Export context as .ovpack file."""
- return await self._service.pack.export_ovpack(uri, to, ctx=self._ctx)
+ return await self._service.pack.export_ovpack(
+ uri,
+ to,
+ ctx=self._ctx,
+ include_vectors=include_vectors,
+ )
- async def backup_ovpack(self, to: str) -> str:
+ async def backup_ovpack(self, to: str, include_vectors: bool = False) -> str:
"""Back up public scopes as a restore-only .ovpack file."""
- return await self._service.pack.backup_ovpack(to, ctx=self._ctx)
+ return await self._service.pack.backup_ovpack(
+ to,
+ ctx=self._ctx,
+ include_vectors=include_vectors,
+ )
async def import_ovpack(
self,
file_path: str,
parent: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Import .ovpack file."""
return await self._service.pack.import_ovpack(
@@ -569,22 +584,32 @@ class LocalClient(BaseClient):
parent,
ctx=self._ctx,
on_conflict=on_conflict,
+ vector_mode=vector_mode,
)
async def restore_ovpack(
self,
file_path: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Restore backup .ovpack file."""
return await self._service.pack.restore_ovpack(
file_path,
ctx=self._ctx,
on_conflict=on_conflict,
+ vector_mode=vector_mode,
)
# ============= Debug =============
+ async def check_consistency(self, uri: str) -> Dict[str, Any]:
+ """Check filesystem/vector-index consistency for a URI subtree."""
+ return await self._service.check_consistency(
+ uri=uri,
+ ctx=self._ctx,
+ )
+
async def health(self) -> bool:
"""Check service health."""
return True # Local service is always healthy if initialized
diff --git a/openviking/core/namespace.py b/openviking/core/namespace.py
index ae9c97591..625d1c226 100644
--- a/openviking/core/namespace.py
+++ b/openviking/core/namespace.py
@@ -99,6 +99,26 @@ def uri_parts(uri: str) -> list[str]:
return [part for part in normalized[len("viking://") :].split("/") if part]
+def uri_depth(uri: str) -> int:
+ """Return the number of normalized Viking URI path segments."""
+ return len(uri_parts(uri))
+
+
+def uri_leaf_name(uri: str) -> str:
+ """Return the final normalized Viking URI path segment."""
+ parts = uri_parts(uri)
+ return parts[-1] if parts else ""
+
+
+def relative_uri_path(root_uri: str, uri: str) -> str:
+ """Return uri's slash-separated path relative to root_uri, or empty when not nested."""
+ root_parts = uri_parts(root_uri)
+ parts = uri_parts(uri)
+ if parts == root_parts or parts[: len(root_parts)] != root_parts:
+ return ""
+ return "/".join(parts[len(root_parts) :])
+
+
def _content_segment_index(parts: tuple[str, ...]) -> Optional[int]:
"""Return the first content segment after a user/agent namespace root."""
if len(parts) < 2 or parts[0] not in _CONTENT_TYPES_BY_SCOPE:
diff --git a/openviking/server/routers/pack.py b/openviking/server/routers/pack.py
index ee0f72f01..a68947755 100644
--- a/openviking/server/routers/pack.py
+++ b/openviking/server/routers/pack.py
@@ -25,7 +25,18 @@ router = APIRouter(prefix="/api/v1/pack", tags=["pack"])
class ExportRequest(BaseModel):
"""Request model for export."""
+ model_config = ConfigDict(extra="forbid")
+
uri: str
+ include_vectors: bool = False
+
+
+class BackupRequest(BaseModel):
+ """Request model for backup export."""
+
+ model_config = ConfigDict(extra="forbid")
+
+ include_vectors: bool = False
class ImportRequest(BaseModel):
@@ -42,6 +53,7 @@ class ImportRequest(BaseModel):
temp_file_id: str
parent: str
on_conflict: Optional[Literal["fail", "overwrite", "skip"]] = None
+ vector_mode: Optional[Literal["auto", "recompute", "require"]] = None
class RestoreRequest(BaseModel):
@@ -51,6 +63,7 @@ class RestoreRequest(BaseModel):
temp_file_id: str
on_conflict: Optional[Literal["fail", "overwrite", "skip"]] = None
+ vector_mode: Optional[Literal["auto", "recompute", "require"]] = None
@router.post("/export")
@@ -72,7 +85,12 @@ async def export_ovpack(
try:
# Export to temp file
- await service.pack.export_ovpack(uri, temp_file, ctx=ctx)
+ await service.pack.export_ovpack(
+ uri,
+ temp_file,
+ ctx=ctx,
+ include_vectors=body.include_vectors,
+ )
# Determine filename from URI
base_name = uri.strip().rstrip("/").split("/")[-1]
@@ -106,6 +124,7 @@ async def export_ovpack(
@require_auth_root_or_admin
async def backup_ovpack(
request: Request,
+ body: BackupRequest | None = None,
ctx: RequestContext = Depends(get_request_context),
):
"""Back up all public OpenViking scopes as a restore-only .ovpack file."""
@@ -114,7 +133,11 @@ async def backup_ovpack(
temp_file = os.path.join(temp_dir, f"backup_{os.urandom(16).hex()}.ovpack")
try:
- await service.pack.backup_ovpack(temp_file, ctx=ctx)
+ await service.pack.backup_ovpack(
+ temp_file,
+ ctx=ctx,
+ include_vectors=bool(body.include_vectors) if body is not None else False,
+ )
def cleanup():
if os.path.exists(temp_file):
@@ -156,6 +179,7 @@ async def import_ovpack(
parent,
ctx=ctx,
on_conflict=body.on_conflict,
+ vector_mode=body.vector_mode,
)
except Exception:
await store.mark_failed(resolved, ctx)
@@ -185,6 +209,7 @@ async def restore_ovpack(
resolved.local_path,
ctx=ctx,
on_conflict=body.on_conflict,
+ vector_mode=body.vector_mode,
)
except Exception:
await store.mark_failed(resolved, ctx)
diff --git a/openviking/server/routers/system.py b/openviking/server/routers/system.py
index b8a73e3e3..ad75d7565 100644
--- a/openviking/server/routers/system.py
+++ b/openviking/server/routers/system.py
@@ -8,6 +8,7 @@ from fastapi import APIRouter, Depends, Request
from fastapi.responses import JSONResponse
from pydantic import BaseModel
+from openviking.core.uri_validation import validate_viking_uri
from openviking.server.auth import get_request_context, resolve_identity
from openviking.server.dependencies import get_service
from openviking.server.identity import AuthMode, RequestContext
@@ -170,6 +171,12 @@ class WaitRequest(BaseModel):
timeout: Optional[float] = None
+class ConsistencyRequest(BaseModel):
+ """Request model for filesystem/vector-index consistency checks."""
+
+ uri: str
+
+
@router.post("/api/v1/system/wait", tags=["system"])
async def wait_processed(
request: WaitRequest,
@@ -179,3 +186,18 @@ async def wait_processed(
service = get_service()
result = await service.resources.wait_processed(timeout=request.timeout)
return Response(status="ok", result=result)
+
+
[email protected]("/api/v1/system/consistency", tags=["system"])
+async def check_consistency(
+ request: ConsistencyRequest,
+ ctx: RequestContext = Depends(get_request_context),
+):
+ """Check filesystem/vector-index consistency for a URI subtree."""
+ service = get_service()
+ uri = validate_viking_uri(request.uri)
+ result = await service.check_consistency(
+ uri=uri,
+ ctx=ctx,
+ )
+ return Response(status="ok", result=result)
diff --git a/openviking/service/core.py b/openviking/service/core.py
index dd7ccea5c..40934e29e 100644
--- a/openviking/service/core.py
+++ b/openviking/service/core.py
@@ -25,6 +25,7 @@ from openviking.service.task_tracker import set_task_tracker
from openviking.session import SessionCompressor, create_session_compressor
from openviking.storage import VikingDBManager
from openviking.storage.collection_schemas import init_context_collection
+from openviking.storage.index_consistency import check_index_consistency
from openviking.storage.queuefs.queue_manager import QueueManager, init_queue_manager
from openviking.storage.transaction import LockManager, init_lock_manager
from openviking.storage.viking_fs import VikingFS, init_viking_fs
@@ -417,6 +418,35 @@ class OpenVikingService:
ctx=effective_ctx,
)
+ async def check_consistency(
+ self,
+ *,
+ uri: str,
+ ctx: RequestContext | None = None,
+ ) -> dict[str, Any]:
+ """Check filesystem/vector-index consistency for a URI subtree."""
+ if not self._initialized:
+ await self.initialize()
+ if not self._viking_fs:
+ raise NotInitializedError("VikingFS")
+
+ effective_ctx = ctx or RequestContext(user=self.user, role=Role.ROOT)
+ entries = await self._viking_fs.tree(
+ uri,
+ show_all_hidden=True,
+ node_limit=None,
+ level_limit=None,
+ ctx=effective_ctx,
+ )
+ report = await check_index_consistency(
+ self._viking_fs,
+ self._vikingdb_manager,
+ uri,
+ entries,
+ effective_ctx,
+ )
+ return report.to_dict()
+
def _ensure_initialized(self) -> None:
"""Ensure service is initialized."""
if not self._initialized:
diff --git a/openviking/service/pack_service.py b/openviking/service/pack_service.py
index adad20fb8..146b1cee3 100644
--- a/openviking/service/pack_service.py
+++ b/openviking/service/pack_service.py
@@ -10,10 +10,10 @@ from typing import Optional
from openviking.core.uri_validation import validate_viking_uri
from openviking.server.identity import RequestContext
-from openviking.storage.local_fs import backup_ovpack as local_backup_ovpack
-from openviking.storage.local_fs import export_ovpack as local_export_ovpack
-from openviking.storage.local_fs import import_ovpack as local_import_ovpack
-from openviking.storage.local_fs import restore_ovpack as local_restore_ovpack
+from openviking.storage.ovpack.operations import backup_ovpack as local_backup_ovpack
+from openviking.storage.ovpack.operations import export_ovpack as local_export_ovpack
+from openviking.storage.ovpack.operations import import_ovpack as local_import_ovpack
+from openviking.storage.ovpack.operations import restore_ovpack as local_restore_ovpack
from openviking.storage.viking_fs import VikingFS
from openviking_cli.exceptions import NotInitializedError
from openviking_cli.utils import get_logger
@@ -43,7 +43,13 @@ class PackService:
raise NotInitializedError("VikingFS")
return self._viking_fs
- async def export_ovpack(self, uri: str, to: str, ctx: RequestContext) -> str:
+ async def export_ovpack(
+ self,
+ uri: str,
+ to: str,
+ ctx: RequestContext,
+ include_vectors: bool = False,
+ ) -> str:
"""Export specified context path as .ovpack file.
Args:
@@ -61,9 +67,15 @@ class PackService:
to,
ctx=ctx,
vector_store=self._vector_store,
+ include_vectors=include_vectors,
)
- async def backup_ovpack(self, to: str, ctx: RequestContext) -> str:
+ async def backup_ovpack(
+ self,
+ to: str,
+ ctx: RequestContext,
+ include_vectors: bool = False,
+ ) -> str:
"""Back up all public OpenViking scopes as a restore-only .ovpack file."""
viking_fs = self._ensure_initialized()
return await local_backup_ovpack(
@@ -71,6 +83,7 @@ class PackService:
to,
ctx=ctx,
vector_store=self._vector_store,
+ include_vectors=include_vectors,
)
async def import_ovpack(
@@ -79,6 +92,7 @@ class PackService:
parent: str,
ctx: RequestContext,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Import local .ovpack file to specified parent path.
@@ -97,6 +111,8 @@ class PackService:
file_path,
parent,
on_conflict=on_conflict,
+ vector_mode=vector_mode,
+ vector_store=self._vector_store,
ctx=ctx,
)
@@ -105,6 +121,7 @@ class PackService:
file_path: str,
ctx: RequestContext,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Restore a backup .ovpack file to its original public scope roots."""
viking_fs = self._ensure_initialized()
@@ -113,4 +130,6 @@ class PackService:
file_path,
ctx=ctx,
on_conflict=on_conflict,
+ vector_mode=vector_mode,
+ vector_store=self._vector_store,
)
diff --git a/openviking/storage/index_consistency.py b/openviking/storage/index_consistency.py
new file mode 100644
index 000000000..67faadbb0
--- /dev/null
+++ b/openviking/storage/index_consistency.py
@@ -0,0 +1,225 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Data consistency checks between VikingFS content and vector index records."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any
+
+from openviking.core.context import ResourceContentType
+from openviking.server.identity import RequestContext
+from openviking.storage.expr import Eq
+from openviking.utils.embedding_utils import get_resource_content_type
+from openviking_cli.utils.logger import get_logger
+from openviking_cli.utils.uri import VikingURI
+
+logger = get_logger(__name__)
+
+NON_INDEX_SCOPES = frozenset({"session"})
+PUBLIC_MISSING_RECORD_LIMIT = 20
+ERROR_DETAILS_MISSING_RECORD_LIMIT = 1
+
+
+@dataclass(frozen=True)
+class IndexExpectation:
+ """One vector index record expected from filesystem content."""
+
+ uri: str
+ rel_path: str
+ level: int
+
+ @property
+ def key(self) -> str:
+ path = self.rel_path or "."
+ return f"{path}#level={self.level}"
+
+ def to_dict(self) -> dict[str, Any]:
+ return {
+ "uri": self.uri,
+ "path": self.rel_path,
+ "level": self.level,
+ "key": self.key,
+ }
+
+
+@dataclass(frozen=True)
+class IndexConsistencyReport:
+ """Result of checking filesystem/index consistency for a subtree."""
+
+ expected: tuple[IndexExpectation, ...]
+ missing_records: tuple[IndexExpectation, ...]
+
+ @property
+ def ok(self) -> bool:
+ return not self.missing_records
+
+ def details(self, limit: int = ERROR_DETAILS_MISSING_RECORD_LIMIT) -> dict[str, Any]:
+ limited = self.missing_records[:limit]
+ return {
+ "expected_count": len(self.expected),
+ "missing_record_count": len(self.missing_records),
+ "missing_records": [item.key for item in limited],
+ "missing_records_truncated": len(self.missing_records) > len(limited),
+ }
+
+ def to_dict(self, limit: int = PUBLIC_MISSING_RECORD_LIMIT) -> dict[str, Any]:
+ limited = self.missing_records[:limit]
+ return {
+ "ok": self.ok,
+ "expected_count": len(self.expected),
+ "missing_record_count": len(self.missing_records),
+ "missing_records": [item.to_dict() for item in limited],
+ "missing_records_truncated": len(self.missing_records) > len(limited),
+ }
+
+
+def _join_uri(base_uri: str, rel_path: str) -> str:
+ return VikingURI(base_uri).join(rel_path).uri
+
+
+def _entry_uri(root_uri: str, entry: dict[str, Any]) -> str:
+ uri = entry.get("uri")
+ if isinstance(uri, str) and uri:
+ return uri
+ rel_path = str(entry.get("rel_path") or "")
+ return _join_uri(root_uri, rel_path)
+
+
+def _is_index_scope(uri: str) -> bool:
+ try:
+ return VikingURI(uri).scope not in NON_INDEX_SCOPES
+ except Exception:
+ return False
+
+
+def _leaf_name(uri_or_path: str) -> str:
+ return uri_or_path.rstrip("/").split("/")[-1]
+
+
+async def _read_text_if_exists(viking_fs, uri: str, ctx: RequestContext) -> str:
+ try:
+ if not await viking_fs.exists(uri, ctx=ctx):
+ return ""
+ content = await viking_fs.read_file(uri, ctx=ctx)
+ return content.decode("utf-8") if isinstance(content, bytes) else str(content)
+ except Exception:
+ return ""
+
+
+def _directory_candidates(
+ root_uri: str,
+ entries: list[dict[str, Any]],
+) -> list[tuple[str, str]]:
+ candidates: list[tuple[str, str]] = []
+ if root_uri != "viking://" and _is_index_scope(root_uri):
+ candidates.append((root_uri, ""))
+
+ for entry in entries:
+ if not entry.get("isDir"):
+ continue
+ uri = _entry_uri(root_uri, entry)
+ if not _is_index_scope(uri):
+ continue
+ rel_path = str(entry.get("rel_path") or "")
+ candidates.append((uri, rel_path))
+ return candidates
+
+
+def _file_candidates(
+ root_uri: str,
+ entries: list[dict[str, Any]],
+) -> list[tuple[str, str, str]]:
+ candidates: list[tuple[str, str, str]] = []
+ for entry in entries:
+ if entry.get("isDir"):
+ continue
+ uri = _entry_uri(root_uri, entry)
+ if not _is_index_scope(uri):
+ continue
+ rel_path = str(entry.get("rel_path") or "")
+ name = str(entry.get("name") or _leaf_name(rel_path))
+ if name.startswith("."):
+ continue
+ candidates.append((uri, rel_path, name))
+ return candidates
+
+
+async def build_index_expectations(
+ viking_fs,
+ root_uri: str,
+ entries: list[dict[str, Any]],
+ ctx: RequestContext,
+) -> tuple[IndexExpectation, ...]:
+ """Build the index records expected from current filesystem content."""
+ expectations: list[IndexExpectation] = []
+
+ for uri, rel_path in _directory_candidates(root_uri, entries):
+ abstract = await _read_text_if_exists(viking_fs, f"{uri}/.abstract.md", ctx)
+ overview = await _read_text_if_exists(viking_fs, f"{uri}/.overview.md", ctx)
+ if abstract:
+ expectations.append(IndexExpectation(uri=uri, rel_path=rel_path, level=0))
+ if overview:
+ expectations.append(IndexExpectation(uri=uri, rel_path=rel_path, level=1))
+
+ for uri, rel_path, name in _file_candidates(root_uri, entries):
+ if get_resource_content_type(name) == ResourceContentType.TEXT:
+ expectations.append(IndexExpectation(uri=uri, rel_path=rel_path, level=2))
+
+ return tuple(sorted(expectations, key=lambda item: (item.rel_path, item.level)))
+
+
+def _record_level(record: dict[str, Any]) -> int | None:
+ level = record.get("level")
+ if isinstance(level, bool) or not isinstance(level, int):
+ return None
+ return level
+
+
+async def _fetch_index_records(vector_store, uri: str, ctx: RequestContext) -> list[dict[str, Any]]:
+ if not vector_store or not hasattr(vector_store, "filter"):
+ return []
+
+ kwargs = {
+ "filter": Eq("uri", uri),
+ "limit": 10,
+ "output_fields": ["uri", "level"],
+ }
+ try:
+ return await vector_store.filter(**kwargs, ctx=ctx)
+ except TypeError:
+ try:
+ return await vector_store.filter(**kwargs)
+ except Exception as exc:
+ logger.warning(f"Failed to check vector index records for {uri}: {exc}")
+ except Exception as exc:
+ logger.warning(f"Failed to check vector index records for {uri}: {exc}")
+ return []
+
+
+async def check_index_consistency(
+ viking_fs,
+ vector_store,
+ root_uri: str,
+ entries: list[dict[str, Any]],
+ ctx: RequestContext,
+) -> IndexConsistencyReport:
+ """Check that filesystem content has the expected vector index records."""
+ expectations = await build_index_expectations(viking_fs, root_uri, entries, ctx)
+ missing_records: list[IndexExpectation] = []
+
+ records_by_uri: dict[str, dict[int, dict[str, Any]]] = {}
+ for expectation in expectations:
+ if expectation.uri not in records_by_uri:
+ records = await _fetch_index_records(vector_store, expectation.uri, ctx)
+ records_by_uri[expectation.uri] = {
+ level: record for record in records if (level := _record_level(record)) is not None
+ }
+ record = records_by_uri[expectation.uri].get(expectation.level)
+ if record is None:
+ missing_records.append(expectation)
+
+ return IndexConsistencyReport(
+ expected=expectations,
+ missing_records=tuple(missing_records),
+ )
diff --git a/openviking/storage/local_fs.py b/openviking/storage/local_fs.py
deleted file mode 100644
index 0731c5ead..000000000
--- a/openviking/storage/local_fs.py
+++ /dev/null
@@ -1,1279 +0,0 @@
-# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
-# SPDX-License-Identifier: AGPL-3.0
-import asyncio
-import hashlib
-import json
-import os
-import re
-import zipfile
-from typing import Any, Optional
-
-from openviking.core.namespace import context_type_for_uri
-from openviking.resource.watch_storage import is_watch_task_control_uri
-from openviking.server.identity import RequestContext
-from openviking.storage.expr import Eq
-from openviking.utils.embedding_utils import vectorize_directory_meta, vectorize_file
-from openviking_cli.exceptions import ConflictError, InvalidArgumentError, NotFoundError
-from openviking_cli.utils.logger import get_logger
-from openviking_cli.utils.uri import VikingURI
-
-logger = get_logger(__name__)
-
-OVPACK_FORMAT_VERSION = 2
-OVPACK_KIND = "openviking.ovpack"
-OVPACK_MANIFEST_FILENAME = ".ovpack_manifest.json"
-OVPACK_MANIFEST_ZIP_LEAF = "_._ovpack_manifest.json"
-OVPACK_ON_CONFLICT_VALUES = frozenset({"fail", "overwrite", "skip"})
-OVPACK_BACKUP_NAME = "openviking-backup"
-OVPACK_BACKUP_TYPE = "backup"
-
-_PUBLIC_SCOPES = ("resources", "user", "agent", "session")
-_IMPORTABLE_SCOPES = frozenset(_PUBLIC_SCOPES)
-_STRUCTURED_IMPORT_SCOPES = frozenset({"user", "agent", "session"})
-_EXCLUDED_FILENAMES = frozenset({".relations.json", OVPACK_MANIFEST_FILENAME})
-_NON_VECTOR_SCOPES = frozenset({"session"})
-_PORTABLE_VECTOR_SCALAR_FIELDS = [
- "uri",
- "type",
- "context_type",
- "level",
- "name",
- "description",
- "tags",
- "abstract",
-]
-
-_UNSAFE_PATH_RE = re.compile(r"(^|[\\/])\.\.($|[\\/])")
-_DRIVE_RE = re.compile(r"^[A-Za-z]:")
-_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
-
-
-def _strip_uri_trailing_slash(uri: str) -> str:
- normalized = VikingURI(uri.strip()).uri
- return normalized if normalized == "viking://" else normalized.rstrip("/")
-
-
-def _join_uri(base_uri: str, rel_path: str) -> str:
- base_uri = _strip_uri_trailing_slash(base_uri)
- if base_uri == "viking://":
- return f"viking://{rel_path}" if rel_path else base_uri
- return f"{base_uri}/{rel_path}" if rel_path else base_uri
-
-
-def _rel_path_for_uri(root_uri: str, uri: str) -> str:
- root_uri = _strip_uri_trailing_slash(root_uri)
- uri = _strip_uri_trailing_slash(uri)
- if uri == root_uri:
- return ""
- prefix = f"{root_uri}/"
- return uri[len(prefix) :] if uri.startswith(prefix) else ""
-
-
-def _leaf_name(uri_or_path: str) -> str:
- return uri_or_path.rstrip("/").split("/")[-1]
-
-
-def _sha256_hex(data: bytes) -> str:
- return hashlib.sha256(data).hexdigest()
-
-
-def _is_excluded_rel_path(rel_path: str) -> bool:
- return _leaf_name(rel_path) in _EXCLUDED_FILENAMES
-
-
-def _is_manifest_zip_path(zip_path: str, base_name: str) -> bool:
- return zip_path == f"{base_name}/{OVPACK_MANIFEST_ZIP_LEAF}"
-
-
-def _validate_ovpack_member_path(zip_path: str, base_name: str) -> str:
- """Validate a zip member path for ovpack imports and reject unsafe entries."""
- if not zip_path:
- raise ValueError("Invalid ovpack entry: empty path")
- if "\\" in zip_path:
- raise ValueError(f"Unsafe ovpack entry path: {zip_path!r}")
- if zip_path.startswith("/"):
- raise ValueError(f"Unsafe ovpack entry path: {zip_path!r}")
- if _DRIVE_RE.match(zip_path):
- raise ValueError(f"Unsafe ovpack entry path: {zip_path!r}")
- if _UNSAFE_PATH_RE.search(zip_path):
- raise ValueError(f"Unsafe ovpack entry path: {zip_path!r}")
-
- parts = zip_path.split("/")
- if any(part == ".." for part in parts):
- raise ValueError(f"Unsafe ovpack entry path: {zip_path!r}")
- if not parts or parts[0] != base_name:
- raise ValueError(f"Invalid ovpack entry root: {zip_path!r}")
-
- return zip_path
-
-
-def ensure_ovpack_extension(path: str) -> str:
- """Ensure path ends with .ovpack extension."""
- if not path.endswith(".ovpack"):
- return path + ".ovpack"
- return path
-
-
-def ensure_dir_exists(path: str) -> None:
- """Ensure the parent directory of the given path exists."""
- out_dir = os.path.dirname(os.path.abspath(path))
- if out_dir:
- os.makedirs(out_dir, exist_ok=True)
-
-
-def get_ovpack_zip_path(base_name: str, rel_path: str) -> str:
- """Generate ZIP internal path from a Viking relative path."""
- if not rel_path:
- return f"{base_name}/"
- parts = rel_path.split("/")
- escaped = [("_._" + part[1:]) if part.startswith(".") else part for part in parts]
- return f"{base_name}/{'/'.join(escaped)}"
-
-
-def get_viking_rel_path_from_zip(zip_path: str) -> str:
- """Restore Viking relative path from ZIP path."""
- parts = zip_path.split("/")
- if len(parts) <= 1:
- return ""
-
- rel_parts = parts[1:]
- restored = [("." + part[3:]) if part.startswith("_._") else part for part in rel_parts]
- return "/".join(restored)
-
-
-def _validate_public_scope(uri: str, *, operation: str, allow_root: bool = False) -> None:
- parsed = VikingURI(uri)
- if parsed.uri == "viking://":
- if allow_root:
- return
- raise InvalidArgumentError(f"ovpack {operation} is not supported for root URI")
- if parsed.scope not in _IMPORTABLE_SCOPES:
- raise InvalidArgumentError(f"ovpack {operation} is not supported for scope: {parsed.scope}")
-
-
-def _validate_import_target_uri(uri: str) -> None:
- """Enforce the same target-policy boundary as direct content writes."""
- _validate_public_scope(uri, operation="import")
- name = _leaf_name(uri)
- if name in _EXCLUDED_FILENAMES:
- raise InvalidArgumentError(f"cannot import internal ovpack file: {uri}")
- if is_watch_task_control_uri(uri):
- raise InvalidArgumentError(f"cannot import watch task control file: {uri}")
-
-
-def _validate_export_source_uri(uri: str) -> None:
- _validate_public_scope(uri, operation="export")
- name = _leaf_name(uri)
- if name in _EXCLUDED_FILENAMES:
- raise InvalidArgumentError(f"cannot export internal ovpack file: {uri}")
- if is_watch_task_control_uri(uri):
- raise InvalidArgumentError(f"cannot export watch task control file: {uri}")
-
-
-def _base_name_from_entries(infolist: list[zipfile.ZipInfo]) -> str:
- for info in infolist:
- filename = info.filename
- if filename:
- base_name = filename.replace("\\", "/").split("/")[0]
- if base_name:
- return base_name
- raise ValueError("Could not determine root directory name from ovpack")
-
-
-def _normalize_on_conflict(on_conflict: Optional[str]) -> str:
- if on_conflict is None:
- return "fail"
- if on_conflict not in OVPACK_ON_CONFLICT_VALUES:
- allowed = ", ".join(sorted(OVPACK_ON_CONFLICT_VALUES))
- raise InvalidArgumentError(
- f"Invalid on_conflict value: {on_conflict}. Must be one of: {allowed}"
- )
- return on_conflict
-
-
-def _portable_scalars(record: dict[str, Any]) -> dict[str, Any]:
- return {
- field: record[field]
- for field in _PORTABLE_VECTOR_SCALAR_FIELDS
- if field != "uri" and record.get(field) is not None
- }
-
-
-def _record_level(record: dict[str, Any], default: int = 2) -> int:
- try:
- return int(record.get("level", default))
- except (TypeError, ValueError):
- return default
-
-
-async def _call_vector_filter(vector_store, uri: str, ctx: RequestContext) -> list[dict[str, Any]]:
- if not vector_store or not hasattr(vector_store, "filter"):
- return []
-
- kwargs = {
- "filter": Eq("uri", uri),
- "limit": 10,
- "output_fields": _PORTABLE_VECTOR_SCALAR_FIELDS,
- }
- try:
- return await vector_store.filter(**kwargs, ctx=ctx)
- except TypeError:
- try:
- return await vector_store.filter(**kwargs)
- except Exception as exc:
- logger.warning(f"Failed to export vector scalars for {uri}: {exc}")
- except Exception as exc:
- logger.warning(f"Failed to export vector scalars for {uri}: {exc}")
- return []
-
-
-async def _read_text_if_exists(viking_fs, uri: str, ctx: RequestContext) -> str:
- try:
- if not await viking_fs.exists(uri, ctx=ctx):
- return ""
- content = await viking_fs.read_file(uri, ctx=ctx)
- return content.decode("utf-8") if isinstance(content, bytes) else content
- except Exception:
- return ""
-
-
-async def _directory_vector_texts(viking_fs, uri: str, ctx: RequestContext) -> dict[int, str]:
- abstract = await _read_text_if_exists(viking_fs, f"{uri}/.abstract.md", ctx)
- overview = await _read_text_if_exists(viking_fs, f"{uri}/.overview.md", ctx)
- return {0: abstract, 1: overview}
-
-
-async def _manifest_vector_records(
- viking_fs,
- vector_store,
- uri: str,
- is_dir: bool,
- ctx: RequestContext,
-) -> list[dict[str, Any]]:
- records = await _call_vector_filter(vector_store, uri, ctx)
- records_by_level = {_record_level(record): record for record in records}
- texts = await _directory_vector_texts(viking_fs, uri, ctx) if is_dir else {}
-
- manifest_records: list[dict[str, Any]] = []
- for level in sorted(records_by_level):
- item = {
- "level": level,
- "scalars": _portable_scalars(records_by_level[level]),
- }
- text = texts.get(level)
- if text:
- item["text"] = text
- manifest_records.append(item)
-
- if is_dir:
- abstract = texts.get(0, "")
- for level, text in texts.items():
- if text and level not in records_by_level:
- manifest_records.append(
- {
- "level": level,
- "text": text,
- "scalars": {
- "context_type": context_type_for_uri(uri),
- "level": level,
- "abstract": abstract,
- },
- }
- )
-
- return sorted(manifest_records, key=lambda item: int(item.get("level", 2)))
-
-
-async def _build_manifest(
- viking_fs,
- vector_store,
- root_uri: str,
- base_name: str,
- entries: list[dict[str, Any]],
- ctx: RequestContext,
- package_type: Optional[str] = None,
- scopes: Optional[list[str]] = None,
-) -> dict[str, Any]:
- manifest_entries = [{"path": "", "kind": "directory"}]
- vectors: dict[str, list[dict[str, Any]]] = {}
-
- if root_uri != "viking://":
- root_vectors = await _manifest_vector_records(
- viking_fs, vector_store, root_uri, is_dir=True, ctx=ctx
- )
- if root_vectors:
- vectors[""] = root_vectors
-
- for entry in entries:
- rel_path = entry["rel_path"]
- is_dir = bool(entry.get("isDir"))
- manifest_entries.append(
- {
- "path": rel_path,
- "kind": "directory" if is_dir else "file",
- "size": entry.get("size", 0) if not is_dir else 0,
- }
- )
- records = await _manifest_vector_records(
- viking_fs,
- vector_store,
- _join_uri(root_uri, rel_path),
- is_dir=is_dir,
- ctx=ctx,
- )
- if records:
- vectors[rel_path] = records
-
- root = {
- "name": base_name,
- "uri": root_uri,
- "scope": "root" if root_uri == "viking://" else VikingURI(root_uri).scope,
- }
- manifest: dict[str, Any] = {
- "kind": OVPACK_KIND,
- "format_version": OVPACK_FORMAT_VERSION,
- "root": root,
- "entries": manifest_entries,
- "vectors": vectors,
- }
- if package_type:
- root["package_type"] = package_type
- if scopes is not None:
- manifest["scopes"] = scopes
- return manifest
-
-
-def _invalid_manifest(message: str, manifest_path: str, **details: Any) -> InvalidArgumentError:
- return InvalidArgumentError(
- message,
- details={"manifest_path": manifest_path, **details},
- )
-
-
-def _read_manifest(zf: zipfile.ZipFile, base_name: str) -> dict[str, Any]:
- manifest_path = f"{base_name}/{OVPACK_MANIFEST_ZIP_LEAF}"
- try:
- raw = zf.read(manifest_path)
- except KeyError:
- raise _invalid_manifest(
- "Missing ovpack manifest",
- manifest_path,
- hint="Re-export this package with OVPack v2 checksum support before importing.",
- )
-
- try:
- manifest = json.loads(raw.decode("utf-8"))
- except json.JSONDecodeError as exc:
- raise _invalid_manifest(
- "Invalid JSON in ovpack manifest",
- manifest_path,
- reason=str(exc),
- ) from exc
- if not isinstance(manifest, dict):
- raise _invalid_manifest(
- "Invalid ovpack manifest",
- manifest_path,
- actual_type=type(manifest).__name__,
- )
-
- version = manifest.get("format_version")
- if version is None:
- raise _invalid_manifest(
- "Missing ovpack format_version",
- manifest_path,
- field="format_version",
- )
- try:
- version_int = int(version)
- except (TypeError, ValueError) as exc:
- raise _invalid_manifest(
- f"Invalid ovpack format_version {version!r}",
- manifest_path,
- field="format_version",
- value=version,
- ) from exc
- if version_int < 1:
- raise _invalid_manifest(
- f"Invalid ovpack format_version {version!r}",
- manifest_path,
- field="format_version",
- value=version,
- )
- if version_int != OVPACK_FORMAT_VERSION:
- raise _invalid_manifest(
- f"Unsupported ovpack format_version {version}; "
- f"this OpenViking requires {OVPACK_FORMAT_VERSION}",
- manifest_path,
- format_version=version_int,
- supported_format_version=OVPACK_FORMAT_VERSION,
- )
- if manifest.get("kind") != OVPACK_KIND:
- raise _invalid_manifest(
- "Invalid ovpack manifest kind",
- manifest_path,
- expected=OVPACK_KIND,
- actual=manifest.get("kind"),
- )
- return manifest
-
-
-def _manifest_root_uri(manifest: dict[str, Any]) -> str:
- root = manifest.get("root")
- if not isinstance(root, dict):
- return ""
- uri = root.get("uri")
- if isinstance(uri, str):
- try:
- return _strip_uri_trailing_slash(uri)
- except Exception:
- return uri.rstrip("/")
- return ""
-
-
-def _is_backup_package(manifest: dict[str, Any]) -> bool:
- root = manifest.get("root")
- return (
- isinstance(root, dict)
- and root.get("package_type") == OVPACK_BACKUP_TYPE
- and _manifest_root_uri(manifest) == "viking://"
- )
-
-
-def _is_top_level_scope_package(base_name: str, manifest: dict[str, Any]) -> bool:
- root_uri = _manifest_root_uri(manifest)
- return base_name in _IMPORTABLE_SCOPES and root_uri == f"viking://{base_name}"
-
-
-def _resolve_import_root_uri(parent: str, base_name: str, manifest: dict[str, Any]) -> str:
- if _is_backup_package(manifest):
- raise InvalidArgumentError(
- "Backup ovpack packages must be restored with ov restore or the restore API",
- details={"root": base_name, "parent": parent},
- )
-
- if parent == "viking://":
- if not _is_top_level_scope_package(base_name, manifest):
- raise InvalidArgumentError(
- "Only top-level scope ovpack packages can be imported to viking://",
- details={"root": base_name},
- )
- return f"viking://{base_name}"
-
- if _is_top_level_scope_package(base_name, manifest):
- raise InvalidArgumentError(
- "Top-level scope ovpack packages must be imported to viking://",
- details={"root": base_name, "parent": parent},
- )
- return _join_uri(parent, base_name)
-
-
-def _parse_import_uri(uri: str, *, field: str) -> VikingURI:
- if not uri:
- raise InvalidArgumentError(f"Missing ovpack {field}")
- try:
- return VikingURI(uri)
- except ValueError as exc:
- raise InvalidArgumentError(f"Invalid ovpack {field}", details={field: uri}) from exc
-
-
-def _uri_depth(parsed: VikingURI) -> int:
- return len([part for part in parsed.full_path.split("/") if part])
-
-
-def _validate_import_scope_compatibility(manifest: dict[str, Any], target_root_uri: str) -> None:
- source_root_uri = _manifest_root_uri(manifest)
- source = _parse_import_uri(source_root_uri, field="manifest root uri")
- target = _parse_import_uri(target_root_uri, field="target root uri")
-
- if source.scope not in _IMPORTABLE_SCOPES:
- raise InvalidArgumentError(
- "ovpack import is not supported for source scope",
- details={"source_scope": source.scope},
- )
- if source.scope != target.scope:
- raise InvalidArgumentError(
- "ovpack source scope does not match target scope",
- details={
- "source_scope": source.scope,
- "target_scope": target.scope,
- },
- )
- if source.scope in _STRUCTURED_IMPORT_SCOPES and _uri_depth(source) != _uri_depth(target):
- raise InvalidArgumentError(
- "ovpack source path is incompatible with target path",
- details={"source": source_root_uri, "target": target_root_uri},
- )
-
-
-def _manifest_records_by_level(
- manifest: dict[str, Any], rel_path: str
-) -> dict[int, dict[str, Any]]:
- vectors = manifest.get("vectors") if isinstance(manifest, dict) else None
- records = vectors.get(rel_path, []) if isinstance(vectors, dict) else []
- if not isinstance(records, list):
- return {}
-
- by_level: dict[int, dict[str, Any]] = {}
- for record in records:
- if not isinstance(record, dict):
- continue
- try:
- level = int(record.get("level", 2))
- except (TypeError, ValueError):
- continue
- by_level[level] = record
- return by_level
-
-
-def _manifest_scalar_overrides(
- manifest: dict[str, Any], rel_path: str
-) -> dict[int, dict[str, Any]]:
- overrides: dict[int, dict[str, Any]] = {}
- for level, record in _manifest_records_by_level(manifest, rel_path).items():
- scalars = record.get("scalars")
- if isinstance(scalars, dict):
- overrides[level] = dict(scalars)
- return overrides
-
-
-def _normalize_sha256(value: Any, *, field: str, path: str | None = None) -> str:
- if not isinstance(value, str) or not _SHA256_RE.fullmatch(value):
- details = {"field": field}
- if path is not None:
- details["path"] = path
- raise InvalidArgumentError(f"Invalid ovpack manifest {field}", details=details)
- return value.lower()
-
-
-def _manifest_entries_by_path(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]:
- if not isinstance(manifest, dict) or "entries" not in manifest:
- return {}
-
- entries = manifest.get("entries")
- if not isinstance(entries, list):
- raise InvalidArgumentError("Invalid ovpack manifest: entries must be a list")
-
- by_path: dict[str, dict[str, Any]] = {}
- for index, entry in enumerate(entries):
- if not isinstance(entry, dict):
- raise InvalidArgumentError(
- "Invalid ovpack manifest entry",
- details={"index": index},
- )
-
- rel_path = entry.get("path")
- kind = entry.get("kind")
- if not isinstance(rel_path, str):
- raise InvalidArgumentError(
- "Invalid ovpack manifest entry path",
- details={"index": index},
- )
- if kind not in {"directory", "file"}:
- raise InvalidArgumentError(
- "Invalid ovpack manifest entry kind",
- details={"path": rel_path, "kind": kind},
- )
- if rel_path in by_path:
- raise InvalidArgumentError(
- "Duplicate ovpack manifest entry",
- details={"path": rel_path},
- )
- by_path[rel_path] = entry
-
- return by_path
-
-
-def _manifest_content_sha256(file_entries_by_path: dict[str, dict[str, Any]]) -> str:
- content_entries: list[dict[str, Any]] = []
- for rel_path, entry in sorted(file_entries_by_path.items()):
- size = entry.get("size")
- if not isinstance(size, int) or isinstance(size, bool) or size < 0:
- raise InvalidArgumentError(
- "Invalid ovpack manifest file size",
- details={"path": rel_path, "size": size},
- )
- content_entries.append(
- {
- "path": rel_path,
- "size": size,
- "sha256": _normalize_sha256(entry.get("sha256"), field="sha256", path=rel_path),
- }
- )
-
- payload = json.dumps(
- content_entries,
- ensure_ascii=False,
- sort_keys=True,
- separators=(",", ":"),
- ).encode("utf-8")
- return _sha256_hex(payload)
-
-
-def _zip_file_members_by_path(
- infolist: list[zipfile.ZipInfo], base_name: str
-) -> dict[str, tuple[zipfile.ZipInfo, str]]:
- files: dict[str, tuple[zipfile.ZipInfo, str]] = {}
- for info in infolist:
- zip_path = info.filename
- if not zip_path:
- continue
- safe_zip_path = _validate_ovpack_member_path(zip_path, base_name)
- if _is_manifest_zip_path(safe_zip_path, base_name) or safe_zip_path.endswith("/"):
- continue
-
- rel_path = get_viking_rel_path_from_zip(safe_zip_path)
- if rel_path in files:
- raise InvalidArgumentError(
- "Duplicate ovpack file entry",
- details={"path": rel_path},
- )
- files[rel_path] = (info, safe_zip_path)
- return files
-
-
-def _zip_directory_members_by_path(infolist: list[zipfile.ZipInfo], base_name: str) -> set[str]:
- directories: set[str] = set()
- for info in infolist:
- zip_path = info.filename
- if not zip_path:
- continue
- safe_zip_path = _validate_ovpack_member_path(zip_path, base_name)
- if not safe_zip_path.endswith("/"):
- continue
-
- rel_path = get_viking_rel_path_from_zip(safe_zip_path.rstrip("/"))
- if rel_path in directories:
- raise InvalidArgumentError(
- "Duplicate ovpack directory entry",
- details={"path": rel_path},
- )
- directories.add(rel_path)
- return directories
-
-
-def _validate_manifest_content(
- zf: zipfile.ZipFile,
- manifest: dict[str, Any],
- infolist: list[zipfile.ZipInfo],
- base_name: str,
-) -> None:
- if "entries" not in manifest:
- raise InvalidArgumentError(
- "Missing ovpack manifest entries",
- details={"field": "entries"},
- )
-
- manifest_entries = _manifest_entries_by_path(manifest)
- manifest_files = {
- rel_path: entry
- for rel_path, entry in manifest_entries.items()
- if entry.get("kind") == "file"
- }
- manifest_directories = {
- rel_path for rel_path, entry in manifest_entries.items() if entry.get("kind") == "directory"
- }
- zip_files = _zip_file_members_by_path(infolist, base_name)
- zip_directories = _zip_directory_members_by_path(infolist, base_name)
-
- missing_files = sorted(set(manifest_files) - set(zip_files))
- unexpected_files = sorted(set(zip_files) - set(manifest_files))
- missing_directories = sorted(manifest_directories - zip_directories)
- unexpected_directories = sorted(zip_directories - manifest_directories)
- if missing_files or unexpected_files or missing_directories or unexpected_directories:
- raise InvalidArgumentError(
- "ovpack entries do not match manifest",
- details={
- "missing_files": missing_files,
- "unexpected_files": unexpected_files,
- "missing_directories": missing_directories,
- "unexpected_directories": unexpected_directories,
- },
- )
-
- expected_content_sha256 = manifest.get("content_sha256")
- if expected_content_sha256 is None:
- raise InvalidArgumentError(
- "Missing ovpack manifest content_sha256",
- details={"field": "content_sha256"},
- )
- expected_content_sha256 = _normalize_sha256(
- expected_content_sha256,
- field="content_sha256",
- )
- actual_content_sha256 = _manifest_content_sha256(manifest_files)
- if actual_content_sha256 != expected_content_sha256:
- raise InvalidArgumentError(
- "ovpack manifest content_sha256 mismatch",
- details={
- "expected": expected_content_sha256,
- "actual": actual_content_sha256,
- },
- )
-
- for rel_path, (_, safe_zip_path) in sorted(zip_files.items()):
- entry = manifest_files[rel_path]
- data = zf.read(safe_zip_path)
-
- expected_size = entry.get("size")
- if expected_size is not None:
- if (
- not isinstance(expected_size, int)
- or isinstance(expected_size, bool)
- or expected_size < 0
- ):
- raise InvalidArgumentError(
- "Invalid ovpack manifest file size",
- details={"path": rel_path, "size": expected_size},
- )
- if len(data) != expected_size:
- raise InvalidArgumentError(
- "ovpack file size does not match manifest",
- details={
- "path": rel_path,
- "expected": expected_size,
- "actual": len(data),
- },
- )
-
- expected_sha256 = entry.get("sha256")
- if expected_sha256 is not None:
- expected_sha256 = _normalize_sha256(
- expected_sha256,
- field="sha256",
- path=rel_path,
- )
- actual_sha256 = _sha256_hex(data)
- if actual_sha256 != expected_sha256:
- raise InvalidArgumentError(
- "ovpack file sha256 does not match manifest",
- details={
- "path": rel_path,
- "expected": expected_sha256,
- "actual": actual_sha256,
- },
- )
-
-
-def _validated_import_members(
- infolist: list[zipfile.ZipInfo], base_name: str, root_uri: str
-) -> list[tuple[zipfile.ZipInfo, str, str, str]]:
- members: list[tuple[zipfile.ZipInfo, str, str, str]] = []
- for info in infolist:
- zip_path = info.filename
- if not zip_path:
- continue
-
- safe_zip_path = _validate_ovpack_member_path(zip_path, base_name)
- if _is_manifest_zip_path(safe_zip_path, base_name):
- members.append((info, safe_zip_path, "manifest", ""))
- continue
-
- kind = "directory" if safe_zip_path.endswith("/") else "file"
- rel_path = get_viking_rel_path_from_zip(
- safe_zip_path.rstrip("/") if kind == "directory" else safe_zip_path
- )
- if root_uri == "viking://" and rel_path == "":
- members.append((info, safe_zip_path, kind, rel_path))
- continue
- target_uri = _join_uri(root_uri, rel_path)
- _validate_import_target_uri(target_uri)
- members.append((info, safe_zip_path, kind, rel_path))
-
- return members
-
-
-async def _root_exists(viking_fs, root_uri: str, ctx: RequestContext) -> bool:
- try:
- await viking_fs.ls(root_uri, ctx=ctx)
- return True
- except NotFoundError:
- return False
- except FileNotFoundError:
- return False
-
-
-async def _ensure_parent_exists(viking_fs, parent: str, ctx: RequestContext) -> None:
- try:
- await viking_fs.stat(parent, ctx=ctx)
- except Exception:
- await viking_fs.mkdir(parent, ctx=ctx)
-
-
-async def _remove_existing_root(viking_fs, root_uri: str, ctx: RequestContext) -> None:
- if not hasattr(viking_fs, "rm"):
- logger.warning(f"[local_fs] Cannot remove existing resource without rm(): {root_uri}")
- return
- try:
- await viking_fs.rm(root_uri, recursive=True, ctx=ctx)
- except NotFoundError:
- return
- except FileNotFoundError:
- return
-
-
-async def _existing_scope_roots(
- viking_fs, scopes: tuple[str, ...], ctx: RequestContext
-) -> list[str]:
- existing: list[str] = []
- for scope in scopes:
- scope_uri = f"viking://{scope}"
- if await _root_exists(viking_fs, scope_uri, ctx):
- existing.append(scope_uri)
- return existing
-
-
-def _exportable_entries(entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
- return [entry for entry in entries if not _is_excluded_rel_path(entry.get("rel_path", ""))]
-
-
-async def _enqueue_direct_vectorization(
- viking_fs,
- uri: str,
- ctx: RequestContext,
- manifest: Optional[dict[str, Any]] = None,
- manifest_path_root_uri: Optional[str] = None,
-) -> None:
- manifest = manifest or {}
- manifest_path_root_uri = manifest_path_root_uri or uri
- entries = await viking_fs.tree(uri, node_limit=None, level_limit=None, ctx=ctx)
- dir_uris = {uri}
- file_entries: list[tuple[str, str, str, str]] = []
- for entry in entries:
- entry_uri = entry.get("uri")
- if not entry_uri:
- continue
- rel_path = entry.get("rel_path") or _rel_path_for_uri(uri, entry_uri)
- manifest_rel_path = _rel_path_for_uri(manifest_path_root_uri, entry_uri)
- if entry.get("isDir"):
- dir_uris.add(entry_uri)
- continue
- name = entry.get("name", "") or _leaf_name(rel_path)
- if name.startswith("."):
- continue
- parent = VikingURI(entry_uri).parent
- if parent:
- file_entries.append((entry_uri, parent.uri, name, manifest_rel_path))
-
- async def index_dir(dir_uri: str) -> None:
- rel_path = _rel_path_for_uri(manifest_path_root_uri, dir_uri)
- records_by_level = _manifest_records_by_level(manifest, rel_path)
- scalar_overrides = _manifest_scalar_overrides(manifest, rel_path)
- abstract = str(records_by_level.get(0, {}).get("text") or "")
- overview = str(records_by_level.get(1, {}).get("text") or "")
-
- if not abstract:
- abstract = await _read_text_if_exists(viking_fs, f"{dir_uri}/.abstract.md", ctx)
- if not overview:
- overview = await _read_text_if_exists(viking_fs, f"{dir_uri}/.overview.md", ctx)
-
- if not abstract and not overview and not scalar_overrides:
- return
- await vectorize_directory_meta(
- dir_uri,
- abstract,
- overview,
- context_type=context_type_for_uri(dir_uri),
- ctx=ctx,
- include_overview=bool(overview),
- scalar_overrides=scalar_overrides,
- )
-
- async def index_file(file_uri: str, parent_uri: str, name: str, rel_path: str) -> None:
- overrides = _manifest_scalar_overrides(manifest, rel_path)
- scalar_override = overrides.get(2) or next(iter(overrides.values()), {})
- summary = str(scalar_override.get("abstract") or "")
- await vectorize_file(
- file_path=file_uri,
- summary_dict={"name": name, "summary": summary},
- parent_uri=parent_uri,
- context_type=context_type_for_uri(file_uri),
- ctx=ctx,
- scalar_override=scalar_override,
- )
-
- await asyncio.gather(*(index_dir(dir_uri) for dir_uri in dir_uris))
- await asyncio.gather(
- *(
- index_file(file_uri, parent_uri, file_name, rel_path)
- for file_uri, parent_uri, file_name, rel_path in file_entries
- )
- )
-
-
-async def import_ovpack(
- viking_fs,
- file_path: str,
- parent: str,
- ctx: RequestContext,
- on_conflict: Optional[str] = None,
-) -> str:
- """
- Import .ovpack file to the specified parent path.
-
- Args:
- viking_fs: VikingFS instance
- file_path: Local .ovpack file path
- parent: Target parent URI (e.g., viking://resources/...)
- on_conflict: One of "fail", "overwrite", or "skip"
-
- Returns:
- Root resource URI after import
- """
- if not os.path.exists(file_path):
- raise FileNotFoundError(f"File not found: {file_path}")
-
- parent = _strip_uri_trailing_slash(parent)
- _validate_public_scope(parent, operation="import", allow_root=True)
- conflict_action = _normalize_on_conflict(on_conflict)
-
- with zipfile.ZipFile(file_path, "r") as zf:
- infolist = zf.infolist()
- if not infolist:
- raise ValueError("Empty ovpack file")
-
- base_name = _base_name_from_entries(infolist)
- manifest = _read_manifest(zf, base_name)
- root_uri = _resolve_import_root_uri(parent, base_name, manifest)
- _validate_import_scope_compatibility(manifest, root_uri)
- _validate_import_target_uri(root_uri)
- manifest_root = manifest.get("root") if isinstance(manifest.get("root"), dict) else {}
- if manifest_root.get("name") and manifest_root.get("name") != base_name:
- logger.warning(
- f"[local_fs] Manifest root name ({manifest_root.get('name')}) "
- f"does not match zip root ({base_name})"
- )
-
- members = _validated_import_members(infolist, base_name, root_uri)
- existing_roots = [root_uri] if await _root_exists(viking_fs, root_uri, ctx) else []
-
- if existing_roots:
- if conflict_action == "skip":
- logger.info(f"[local_fs] Skipped existing resource at {root_uri}")
- return root_uri
- if conflict_action == "fail":
- resource = existing_roots[0]
- raise ConflictError(
- f"Resource already exists at {resource}. "
- "Use on_conflict='overwrite' to replace it.",
- resource=resource,
- )
-
- _validate_manifest_content(zf, manifest, infolist, base_name)
- if parent != "viking://":
- await _ensure_parent_exists(viking_fs, parent, ctx)
-
- for existing_root in existing_roots:
- logger.info(f"[local_fs] Overwriting existing resource at {existing_root}")
- await _remove_existing_root(viking_fs, existing_root, ctx)
-
- for _, safe_zip_path, kind, rel_path in members:
- if kind == "manifest":
- continue
- if kind == "directory":
- await viking_fs.mkdir(_join_uri(root_uri, rel_path), exist_ok=True, ctx=ctx)
- continue
-
- target_file_uri = _join_uri(root_uri, rel_path)
- data = zf.read(safe_zip_path)
- await viking_fs.write_file_bytes(target_file_uri, data, ctx=ctx)
-
- logger.info(f"[local_fs] Successfully imported {file_path} to {root_uri}")
-
- if VikingURI(root_uri).scope not in _NON_VECTOR_SCOPES:
- await _enqueue_direct_vectorization(viking_fs, root_uri, ctx=ctx, manifest=manifest)
- logger.info(f"[local_fs] Enqueued direct vectorization for: {root_uri}")
- else:
- logger.info(f"[local_fs] Skipped vectorization for non-vector scope: {root_uri}")
-
- return root_uri
-
-
-async def _backup_entries(viking_fs, ctx: RequestContext) -> list[dict[str, Any]]:
- entries: list[dict[str, Any]] = []
- for scope in _PUBLIC_SCOPES:
- scope_uri = f"viking://{scope}"
- entries.append(
- {
- "rel_path": scope,
- "uri": scope_uri,
- "isDir": True,
- "size": 0,
- }
- )
- try:
- scope_entries = await viking_fs.tree(
- scope_uri,
- show_all_hidden=True,
- node_limit=None,
- level_limit=None,
- ctx=ctx,
- )
- except (NotFoundError, FileNotFoundError):
- continue
-
- for entry in _exportable_entries(scope_entries):
- rel_path = entry.get("rel_path", "")
- if not rel_path:
- continue
- scoped_entry = dict(entry)
- scoped_entry["rel_path"] = f"{scope}/{rel_path}"
- scoped_entry["uri"] = _join_uri(scope_uri, rel_path)
- entries.append(scoped_entry)
- return entries
-
-
-def _backup_scopes_from_manifest(
- manifest: dict[str, Any], manifest_entries: dict[str, dict[str, Any]]
-) -> tuple[str, ...]:
- roots = {rel_path.split("/", 1)[0] for rel_path in manifest_entries if rel_path}
- unexpected = sorted(root for root in roots if root not in _IMPORTABLE_SCOPES)
- if unexpected:
- raise InvalidArgumentError(
- "Backup ovpack contains unsupported roots",
- details={"roots": unexpected},
- )
-
- directory_scope_roots = {
- rel_path
- for rel_path, entry in manifest_entries.items()
- if rel_path in _IMPORTABLE_SCOPES and entry.get("kind") == "directory"
- }
- missing_scope_directories = sorted(roots - directory_scope_roots)
- if missing_scope_directories:
- raise InvalidArgumentError(
- "Backup ovpack scope roots must be directory entries",
- details={"missing_scope_directories": missing_scope_directories},
- )
-
- entry_scopes = tuple(scope for scope in _PUBLIC_SCOPES if scope in directory_scope_roots)
- declared_scopes = manifest.get("scopes")
- if not isinstance(declared_scopes, list) or any(
- not isinstance(scope, str) for scope in declared_scopes
- ):
- raise InvalidArgumentError(
- "Invalid backup ovpack scopes",
- details={"field": "scopes"},
- )
- duplicate_scopes = sorted(
- scope for scope in set(declared_scopes) if declared_scopes.count(scope) > 1
- )
- invalid_scopes = sorted(scope for scope in declared_scopes if scope not in _IMPORTABLE_SCOPES)
- if duplicate_scopes or invalid_scopes:
- raise InvalidArgumentError(
- "Invalid backup ovpack scopes",
- details={
- "duplicate_scopes": duplicate_scopes,
- "invalid_scopes": invalid_scopes,
- },
- )
- if set(declared_scopes) != set(entry_scopes):
- raise InvalidArgumentError(
- "Backup ovpack scopes do not match entries",
- details={
- "declared_scopes": declared_scopes,
- "entry_scopes": list(entry_scopes),
- },
- )
- return entry_scopes
-
-
-async def _write_ovpack_archive(
- viking_fs,
- root_uri: str,
- to: str,
- base_name: str,
- entries: list[dict[str, Any]],
- manifest: dict[str, Any],
- ctx: RequestContext,
-) -> str:
- ensure_dir_exists(to)
- manifest_entries = _manifest_entries_by_path(manifest)
- manifest_file_entries = {
- rel_path: entry
- for rel_path, entry in manifest_entries.items()
- if entry.get("kind") == "file"
- }
-
- with zipfile.ZipFile(to, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as zf:
- zf.writestr(base_name + "/", "")
-
- for entry in entries:
- rel_path = entry["rel_path"]
- zip_path = get_ovpack_zip_path(base_name, rel_path)
-
- if entry.get("isDir"):
- zf.writestr(zip_path + "/", "")
- else:
- full_uri = entry.get("uri") or _join_uri(root_uri, rel_path)
- try:
- data = await viking_fs.read_file_bytes(full_uri, ctx=ctx)
- except Exception as exc:
- logger.warning(f"Failed to export file {full_uri}: {exc}")
- raise
-
- manifest_entry = manifest_file_entries.get(rel_path)
- if manifest_entry is not None:
- manifest_entry["size"] = len(data)
- manifest_entry["sha256"] = _sha256_hex(data)
- zf.writestr(zip_path, data)
-
- manifest["content_sha256"] = _manifest_content_sha256(manifest_file_entries)
- zf.writestr(
- f"{base_name}/{OVPACK_MANIFEST_ZIP_LEAF}",
- json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2).encode("utf-8"),
- )
- return to
-
-
-async def export_ovpack(
- viking_fs,
- uri: str,
- to: str,
- ctx: RequestContext,
- vector_store=None,
-) -> str:
- """
- Export the specified context path as a .ovpack file.
-
- Args:
- viking_fs: VikingFS instance
- uri: Viking URI
- to: Target file path (can be an existing directory or a path ending with .ovpack)
- vector_store: Optional vector store used to export portable scalar metadata
-
- Returns:
- Exported file path
-
- """
- uri = _strip_uri_trailing_slash(uri)
- _validate_export_source_uri(uri)
-
- base_name = _leaf_name(uri) or "export"
-
- if os.path.isdir(to):
- to = os.path.join(to, f"{base_name}.ovpack")
- else:
- to = ensure_ovpack_extension(to)
-
- entries = _exportable_entries(
- await viking_fs.tree(
- uri,
- show_all_hidden=True,
- node_limit=None,
- level_limit=None,
- ctx=ctx,
- )
- )
- manifest = await _build_manifest(viking_fs, vector_store, uri, base_name, entries, ctx)
- await _write_ovpack_archive(viking_fs, uri, to, base_name, entries, manifest, ctx)
-
- logger.info(f"[local_fs] Exported {uri} to {to}")
- return to
-
-
-async def backup_ovpack(
- viking_fs,
- to: str,
- ctx: RequestContext,
- vector_store=None,
-) -> str:
- """Export all public OpenViking scopes as a restore-only backup package."""
- base_name = OVPACK_BACKUP_NAME
- if os.path.isdir(to):
- to = os.path.join(to, f"{base_name}.ovpack")
- else:
- to = ensure_ovpack_extension(to)
-
- entries = await _backup_entries(viking_fs, ctx)
- manifest = await _build_manifest(
- viking_fs,
- vector_store,
- "viking://",
- base_name,
- entries,
- ctx,
- package_type=OVPACK_BACKUP_TYPE,
- scopes=list(_PUBLIC_SCOPES),
- )
- await _write_ovpack_archive(viking_fs, "viking://", to, base_name, entries, manifest, ctx)
-
- logger.info(f"[local_fs] Backed up OpenViking public scopes to {to}")
- return to
-
-
-async def restore_ovpack(
- viking_fs,
- file_path: str,
- ctx: RequestContext,
- on_conflict: Optional[str] = None,
-) -> str:
- """Restore a backup package to its original public scope roots."""
- if not os.path.exists(file_path):
- raise FileNotFoundError(f"File not found: {file_path}")
-
- conflict_action = _normalize_on_conflict(on_conflict)
- root_uri = "viking://"
-
- with zipfile.ZipFile(file_path, "r") as zf:
- infolist = zf.infolist()
- if not infolist:
- raise ValueError("Empty ovpack file")
-
- base_name = _base_name_from_entries(infolist)
- manifest = _read_manifest(zf, base_name)
- if not _is_backup_package(manifest):
- raise InvalidArgumentError(
- "Only backup ovpack packages can be restored with ov restore or the restore API",
- details={"root": base_name},
- )
-
- manifest_entries = _manifest_entries_by_path(manifest)
- backup_scopes = _backup_scopes_from_manifest(manifest, manifest_entries)
- members = _validated_import_members(infolist, base_name, root_uri)
- existing_roots = await _existing_scope_roots(viking_fs, backup_scopes, ctx)
-
- if existing_roots:
- if conflict_action == "skip":
- logger.info("[local_fs] Skipped backup restore because target scopes exist")
- return root_uri
- if conflict_action == "fail":
- resource = existing_roots[0]
- raise ConflictError(
- f"Resource already exists at {resource}. "
- "Use on_conflict='overwrite' to replace it.",
- resource=resource,
- )
-
- _validate_manifest_content(zf, manifest, infolist, base_name)
-
- for existing_root in existing_roots:
- logger.info(f"[local_fs] Overwriting existing resource at {existing_root}")
- await _remove_existing_root(viking_fs, existing_root, ctx)
-
- for _, safe_zip_path, kind, rel_path in members:
- if kind == "manifest" or rel_path == "":
- continue
- if kind == "directory":
- await viking_fs.mkdir(_join_uri(root_uri, rel_path), exist_ok=True, ctx=ctx)
- continue
-
- data = zf.read(safe_zip_path)
- await viking_fs.write_file_bytes(_join_uri(root_uri, rel_path), data, ctx=ctx)
-
- logger.info(f"[local_fs] Successfully restored backup {file_path}")
-
- for scope in backup_scopes:
- if scope in _NON_VECTOR_SCOPES:
- logger.info(f"[local_fs] Skipped vectorization for non-vector scope: {scope}")
- continue
- scope_uri = f"viking://{scope}"
- await _enqueue_direct_vectorization(
- viking_fs,
- scope_uri,
- ctx=ctx,
- manifest=manifest,
- manifest_path_root_uri=root_uri,
- )
- logger.info(f"[local_fs] Enqueued direct vectorization for: {scope_uri}")
-
- return root_uri
diff --git a/openviking/storage/ovpack/__init__.py b/openviking/storage/ovpack/__init__.py
new file mode 100644
index 000000000..ee9955ec7
--- /dev/null
+++ b/openviking/storage/ovpack/__init__.py
@@ -0,0 +1,3 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""OVPack internal helpers."""
diff --git a/openviking/storage/ovpack/format.py b/openviking/storage/ovpack/format.py
new file mode 100644
index 000000000..b0666cb1c
--- /dev/null
+++ b/openviking/storage/ovpack/format.py
@@ -0,0 +1,220 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""Low-level OVPack format helpers.
+
+This module contains only stateless helpers for names, ZIP paths, checksums,
+and binary encodings. It does not know about VikingFS writes, vector stores, or
+import/export conflict handling.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import re
+import struct
+from typing import Any, Optional
+
+from openviking_cli.exceptions import InvalidArgumentError
+from openviking_cli.utils.uri import VikingURI
+
+OVPACK_FORMAT_VERSION = 2
+OVPACK_KIND = "openviking.ovpack"
+OVPACK_INTERNAL_DIR = "_ovpack"
+OVPACK_FILES_DIR = "files"
+OVPACK_MANIFEST_FILENAME = "manifest.json"
+OVPACK_MANIFEST_ZIP_LEAF = f"{OVPACK_INTERNAL_DIR}/{OVPACK_MANIFEST_FILENAME}"
+OVPACK_INDEX_RECORDS_FILENAME = "index_records.jsonl"
+OVPACK_INDEX_RECORDS_PATH = f"{OVPACK_INTERNAL_DIR}/{OVPACK_INDEX_RECORDS_FILENAME}"
+OVPACK_DENSE_FILENAME = "dense.f32"
+OVPACK_DENSE_PATH = f"{OVPACK_INTERNAL_DIR}/{OVPACK_DENSE_FILENAME}"
+OVPACK_ON_CONFLICT_VALUES = frozenset({"fail", "overwrite", "skip"})
+OVPACK_VECTOR_MODE_VALUES = frozenset({"auto", "recompute", "require"})
+OVPACK_BACKUP_NAME = "openviking-backup"
+OVPACK_BACKUP_TYPE = "backup"
+
+_UNSAFE_PATH_RE = re.compile(r"(^|[\\/])\.\.($|[\\/])")
+_DRIVE_RE = re.compile(r"^[A-Za-z]:")
+_SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$")
+
+
+def strip_uri_trailing_slash(uri: str) -> str:
+ normalized = VikingURI(uri.strip()).uri
+ return normalized if normalized == "viking://" else normalized.rstrip("/")
+
+
+def join_uri(base_uri: str, rel_path: str) -> str:
+ return VikingURI(strip_uri_trailing_slash(base_uri)).join(rel_path).uri
+
+
+def leaf_name(uri_or_path: str) -> str:
+ return uri_or_path.rstrip("/").split("/")[-1]
+
+
+def sha256_hex(data: bytes) -> str:
+ return hashlib.sha256(data).hexdigest()
+
+
+def is_manifest_zip_path(zip_path: str, base_name: str) -> bool:
+ return zip_path == f"{base_name}/{OVPACK_MANIFEST_ZIP_LEAF}"
+
+
+def is_internal_zip_path(zip_path: str, base_name: str) -> bool:
+ return zip_path == f"{base_name}/{OVPACK_INTERNAL_DIR}" or zip_path.startswith(
+ f"{base_name}/{OVPACK_INTERNAL_DIR}/"
+ )
+
+
+def is_content_zip_path(zip_path: str, base_name: str) -> bool:
+ return zip_path == f"{base_name}/{OVPACK_FILES_DIR}" or zip_path.startswith(
+ f"{base_name}/{OVPACK_FILES_DIR}/"
+ )
+
+
+def internal_zip_path(base_name: str, internal_path: str) -> str:
+ return f"{base_name}/{internal_path}"
+
+
+def validate_ovpack_rel_path(rel_path: str, *, allow_root: bool = True) -> None:
+ """Validate a user relative path stored under the ZIP files/ namespace."""
+ if rel_path == "":
+ if allow_root:
+ return
+ raise InvalidArgumentError("Invalid ovpack relative path", details={"path": rel_path})
+ if "\\" in rel_path or rel_path.startswith("/") or _DRIVE_RE.match(rel_path):
+ raise InvalidArgumentError("Unsafe ovpack relative path", details={"path": rel_path})
+ if _UNSAFE_PATH_RE.search(rel_path):
+ raise InvalidArgumentError("Unsafe ovpack relative path", details={"path": rel_path})
+ parts = rel_path.split("/")
+ if any(part in {"", ".", ".."} for part in parts):
+ raise InvalidArgumentError("Unsafe ovpack relative path", details={"path": rel_path})
+
+
+def validate_ovpack_member_path(zip_path: str, base_name: str) -> str:
+ """Validate a zip member path for ovpack imports and reject unsafe entries."""
+ if not zip_path:
+ raise ValueError("Invalid ovpack entry: empty path")
+ if "\\" in zip_path:
+ raise ValueError(f"Unsafe ovpack entry path: {zip_path!r}")
+ if zip_path.startswith("/"):
+ raise ValueError(f"Unsafe ovpack entry path: {zip_path!r}")
+ if _DRIVE_RE.match(zip_path):
+ raise ValueError(f"Unsafe ovpack entry path: {zip_path!r}")
+ if _UNSAFE_PATH_RE.search(zip_path):
+ raise ValueError(f"Unsafe ovpack entry path: {zip_path!r}")
+
+ parts = zip_path.split("/")
+ if any(part == ".." for part in parts):
+ raise ValueError(f"Unsafe ovpack entry path: {zip_path!r}")
+ if not parts or parts[0] != base_name:
+ raise ValueError(f"Invalid ovpack entry root: {zip_path!r}")
+
+ return zip_path
+
+
+def ensure_ovpack_extension(path: str) -> str:
+ """Ensure path ends with .ovpack extension."""
+ if not path.endswith(".ovpack"):
+ return path + ".ovpack"
+ return path
+
+
+def ensure_dir_exists(path: str) -> None:
+ """Ensure the parent directory of the given path exists."""
+ out_dir = os.path.dirname(os.path.abspath(path))
+ if out_dir:
+ os.makedirs(out_dir, exist_ok=True)
+
+
+def get_ovpack_zip_path(base_name: str, rel_path: str) -> str:
+ """Generate ZIP internal path from a Viking relative path."""
+ validate_ovpack_rel_path(rel_path)
+ if not rel_path:
+ return f"{base_name}/{OVPACK_FILES_DIR}/"
+ return f"{base_name}/{OVPACK_FILES_DIR}/{rel_path}"
+
+
+def get_viking_rel_path_from_zip(zip_path: str) -> str:
+ """Restore Viking relative path from ZIP path."""
+ parts = zip_path.split("/")
+ if len(parts) < 2 or parts[1] != OVPACK_FILES_DIR:
+ raise ValueError(f"Invalid ovpack content path: {zip_path!r}")
+ if len(parts) == 2:
+ return ""
+ rel_path = "/".join(parts[2:])
+ validate_ovpack_rel_path(rel_path)
+ return rel_path
+
+
+def normalize_on_conflict(on_conflict: Optional[str]) -> str:
+ if on_conflict is None:
+ return "fail"
+ if on_conflict not in OVPACK_ON_CONFLICT_VALUES:
+ allowed = ", ".join(sorted(OVPACK_ON_CONFLICT_VALUES))
+ raise InvalidArgumentError(
+ f"Invalid on_conflict value: {on_conflict}. Must be one of: {allowed}"
+ )
+ return on_conflict
+
+
+def normalize_vector_mode(vector_mode: Optional[str]) -> str:
+ if vector_mode is None:
+ return "auto"
+ if vector_mode not in OVPACK_VECTOR_MODE_VALUES:
+ allowed = ", ".join(sorted(OVPACK_VECTOR_MODE_VALUES))
+ raise InvalidArgumentError(
+ f"Invalid vector_mode value: {vector_mode}. Must be one of: {allowed}"
+ )
+ return vector_mode
+
+
+def normalize_sha256(value: Any, *, field: str, path: str | None = None) -> str:
+ if not isinstance(value, str) or not _SHA256_RE.fullmatch(value):
+ details = {"field": field}
+ if path is not None:
+ details["path"] = path
+ raise InvalidArgumentError(f"Invalid ovpack manifest {field}", details=details)
+ return value.lower()
+
+
+def manifest_content_sha256(file_entries_by_path: dict[str, dict[str, Any]]) -> str:
+ content_entries: list[dict[str, Any]] = []
+ for rel_path, entry in sorted(file_entries_by_path.items()):
+ size = entry.get("size")
+ if not isinstance(size, int) or isinstance(size, bool) or size < 0:
+ raise InvalidArgumentError(
+ "Invalid ovpack manifest file size",
+ details={"path": rel_path, "size": size},
+ )
+ content_entries.append(
+ {
+ "path": rel_path,
+ "size": size,
+ "sha256": normalize_sha256(entry.get("sha256"), field="sha256", path=rel_path),
+ }
+ )
+
+ payload = json.dumps(
+ content_entries,
+ ensure_ascii=False,
+ sort_keys=True,
+ separators=(",", ":"),
+ ).encode("utf-8")
+ return sha256_hex(payload)
+
+
+def jsonl_bytes(records: list[dict[str, Any]]) -> bytes:
+ if not records:
+ return b""
+ lines = [
+ json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+ for record in records
+ ]
+ return ("\n".join(lines) + "\n").encode("utf-8")
+
+
+def dense_values_bytes(values: list[float]) -> bytes:
+ if not values:
+ return b""
+ return struct.pack(f"<{len(values)}f", *values)
diff --git a/openviking/storage/ovpack/index.py b/openviking/storage/ovpack/index.py
new file mode 100644
index 000000000..40c213f4c
--- /dev/null
+++ b/openviking/storage/ovpack/index.py
@@ -0,0 +1,252 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""OVPack index record construction helpers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from openviking.core.namespace import context_type_for_uri
+from openviking.server.identity import RequestContext
+from openviking.storage.expr import Eq
+from openviking.storage.ovpack.format import (
+ OVPACK_FORMAT_VERSION,
+ OVPACK_KIND,
+ join_uri,
+)
+from openviking_cli.utils.logger import get_logger
+from openviking_cli.utils.uri import VikingURI
+
+logger = get_logger(__name__)
+
+PORTABLE_VECTOR_SCALAR_FIELDS = [
+ "uri",
+ "type",
+ "context_type",
+ "level",
+ "name",
+ "description",
+ "tags",
+ "abstract",
+]
+EXPORT_VECTOR_FIELDS = [*PORTABLE_VECTOR_SCALAR_FIELDS, "vector"]
+
+
+def portable_scalars(record: dict[str, Any]) -> dict[str, Any]:
+ return {
+ field: record[field]
+ for field in PORTABLE_VECTOR_SCALAR_FIELDS
+ if field != "uri" and record.get(field) is not None
+ }
+
+
+def record_level(record: dict[str, Any], default: int = 2) -> int:
+ try:
+ return int(record.get("level", default))
+ except (TypeError, ValueError):
+ return default
+
+
+def coerce_dense_vector(value: Any) -> list[float]:
+ if not isinstance(value, list) or not value:
+ return []
+ dense: list[float] = []
+ for item in value:
+ if isinstance(item, bool) or not isinstance(item, (int, float)):
+ return []
+ dense.append(float(item))
+ return dense
+
+
+async def call_vector_filter(
+ vector_store,
+ uri: str,
+ ctx: RequestContext,
+ *,
+ include_vectors: bool = False,
+) -> list[dict[str, Any]]:
+ if not vector_store or not hasattr(vector_store, "filter"):
+ return []
+
+ kwargs = {
+ "filter": Eq("uri", uri),
+ "limit": 10,
+ "output_fields": EXPORT_VECTOR_FIELDS if include_vectors else PORTABLE_VECTOR_SCALAR_FIELDS,
+ }
+ try:
+ return await vector_store.filter(**kwargs, ctx=ctx)
+ except TypeError:
+ try:
+ return await vector_store.filter(**kwargs)
+ except Exception as exc:
+ logger.warning(f"Failed to export vector scalars for {uri}: {exc}")
+ except Exception as exc:
+ logger.warning(f"Failed to export vector scalars for {uri}: {exc}")
+ return []
+
+
+async def read_text_if_exists(viking_fs, uri: str, ctx: RequestContext) -> str:
+ try:
+ if not await viking_fs.exists(uri, ctx=ctx):
+ return ""
+ content = await viking_fs.read_file(uri, ctx=ctx)
+ return content.decode("utf-8") if isinstance(content, bytes) else content
+ except Exception:
+ return ""
+
+
+async def directory_vector_texts(viking_fs, uri: str, ctx: RequestContext) -> dict[int, str]:
+ abstract = await read_text_if_exists(viking_fs, f"{uri}/.abstract.md", ctx)
+ overview = await read_text_if_exists(viking_fs, f"{uri}/.overview.md", ctx)
+ return {0: abstract, 1: overview}
+
+
+async def index_records_for_uri(
+ viking_fs,
+ vector_store,
+ uri: str,
+ is_dir: bool,
+ ctx: RequestContext,
+ *,
+ include_vectors: bool = False,
+) -> list[dict[str, Any]]:
+ records = await call_vector_filter(vector_store, uri, ctx, include_vectors=include_vectors)
+ records_by_level = {record_level(record): record for record in records}
+ texts = await directory_vector_texts(viking_fs, uri, ctx) if is_dir else {}
+
+ index_records: list[dict[str, Any]] = []
+ for level in sorted(records_by_level):
+ source_record = records_by_level[level]
+ item = {
+ "level": level,
+ "scalars": portable_scalars(source_record),
+ }
+ text = texts.get(level)
+ if text:
+ item["text"] = text
+ if include_vectors:
+ dense = coerce_dense_vector(source_record.get("vector"))
+ if dense:
+ item["_dense_vector"] = dense
+ index_records.append(item)
+
+ if is_dir:
+ abstract = texts.get(0, "")
+ for level, text in texts.items():
+ if text and level not in records_by_level:
+ index_records.append(
+ {
+ "level": level,
+ "text": text,
+ "scalars": {
+ "context_type": context_type_for_uri(uri),
+ "level": level,
+ "abstract": abstract,
+ },
+ }
+ )
+
+ return sorted(index_records, key=lambda item: int(item.get("level", 2)))
+
+
+def append_index_records(
+ index_records: list[dict[str, Any]],
+ dense_values: list[float],
+ rel_path: str,
+ kind: str,
+ records: list[dict[str, Any]],
+) -> None:
+ for record in records:
+ item = {
+ "record_id": f"r{len(index_records) + 1:06d}",
+ "path": rel_path,
+ "kind": kind,
+ "level": int(record.get("level", 2)),
+ }
+ text = record.get("text")
+ if isinstance(text, str) and text:
+ item["text"] = text
+ scalars = record.get("scalars")
+ if isinstance(scalars, dict) and scalars:
+ item["scalars"] = dict(scalars)
+
+ dense = coerce_dense_vector(record.get("_dense_vector"))
+ if dense:
+ item["vector"] = {
+ "dense": {
+ "offset": len(dense_values),
+ "dimensions": len(dense),
+ }
+ }
+ dense_values.extend(dense)
+ index_records.append(item)
+
+
+async def build_manifest(
+ viking_fs,
+ vector_store,
+ root_uri: str,
+ base_name: str,
+ entries: list[dict[str, Any]],
+ ctx: RequestContext,
+ package_type: str | None = None,
+ scopes: list[str] | None = None,
+ include_vectors: bool = False,
+) -> tuple[dict[str, Any], list[dict[str, Any]], list[float]]:
+ manifest_entries = [{"path": "", "kind": "directory"}]
+ index_records: list[dict[str, Any]] = []
+ dense_values: list[float] = []
+
+ if root_uri != "viking://":
+ root_records = await index_records_for_uri(
+ viking_fs,
+ vector_store,
+ root_uri,
+ is_dir=True,
+ ctx=ctx,
+ include_vectors=include_vectors,
+ )
+ append_index_records(index_records, dense_values, "", "directory", root_records)
+
+ for entry in entries:
+ rel_path = entry["rel_path"]
+ is_dir = bool(entry.get("isDir"))
+ manifest_entries.append(
+ {
+ "path": rel_path,
+ "kind": "directory" if is_dir else "file",
+ "size": entry.get("size", 0) if not is_dir else 0,
+ }
+ )
+ records = await index_records_for_uri(
+ viking_fs,
+ vector_store,
+ join_uri(root_uri, rel_path),
+ is_dir=is_dir,
+ ctx=ctx,
+ include_vectors=include_vectors,
+ )
+ append_index_records(
+ index_records,
+ dense_values,
+ rel_path,
+ "directory" if is_dir else "file",
+ records,
+ )
+
+ root = {
+ "name": base_name,
+ "uri": root_uri,
+ "scope": "root" if root_uri == "viking://" else VikingURI(root_uri).scope,
+ }
+ manifest: dict[str, Any] = {
+ "kind": OVPACK_KIND,
+ "format_version": OVPACK_FORMAT_VERSION,
+ "root": root,
+ "entries": manifest_entries,
+ }
+ if package_type:
+ root["package_type"] = package_type
+ if scopes is not None:
+ manifest["scopes"] = scopes
+ return manifest, index_records, dense_values
diff --git a/openviking/storage/ovpack/manifest.py b/openviking/storage/ovpack/manifest.py
new file mode 100644
index 000000000..7c70b34c3
--- /dev/null
+++ b/openviking/storage/ovpack/manifest.py
@@ -0,0 +1,227 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""OVPack manifest parsing and structural validation helpers."""
+
+from __future__ import annotations
+
+import json
+import zipfile
+from typing import Any
+
+from openviking.core.namespace import uri_leaf_name
+from openviking.storage.ovpack.format import (
+ OVPACK_DENSE_PATH,
+ OVPACK_FORMAT_VERSION,
+ OVPACK_INDEX_RECORDS_PATH,
+ OVPACK_KIND,
+ OVPACK_MANIFEST_ZIP_LEAF,
+ normalize_sha256,
+ validate_ovpack_rel_path,
+)
+from openviking.storage.ovpack.policy import manifest_root_uri
+from openviking_cli.exceptions import InvalidArgumentError
+
+
+def invalid_manifest(message: str, manifest_path: str, **details: Any) -> InvalidArgumentError:
+ return InvalidArgumentError(
+ message,
+ details={"manifest_path": manifest_path, **details},
+ )
+
+
+def read_manifest(zf: zipfile.ZipFile, base_name: str) -> dict[str, Any]:
+ manifest_path = f"{base_name}/{OVPACK_MANIFEST_ZIP_LEAF}"
+ try:
+ raw = zf.read(manifest_path)
+ except KeyError:
+ raise invalid_manifest(
+ "Missing ovpack manifest",
+ manifest_path,
+ hint="Re-export this package with OVPack v2 checksum support before importing.",
+ )
+
+ try:
+ manifest = json.loads(raw.decode("utf-8"))
+ except json.JSONDecodeError as exc:
+ raise invalid_manifest(
+ "Invalid JSON in ovpack manifest",
+ manifest_path,
+ reason=str(exc),
+ ) from exc
+ if not isinstance(manifest, dict):
+ raise invalid_manifest(
+ "Invalid ovpack manifest",
+ manifest_path,
+ actual_type=type(manifest).__name__,
+ )
+
+ version = manifest.get("format_version")
+ if version is None:
+ raise invalid_manifest(
+ "Missing ovpack format_version",
+ manifest_path,
+ field="format_version",
+ )
+ try:
+ version_int = int(version)
+ except (TypeError, ValueError) as exc:
+ raise invalid_manifest(
+ f"Invalid ovpack format_version {version!r}",
+ manifest_path,
+ field="format_version",
+ value=version,
+ ) from exc
+ if version_int < 1:
+ raise invalid_manifest(
+ f"Invalid ovpack format_version {version!r}",
+ manifest_path,
+ field="format_version",
+ value=version,
+ )
+ if version_int != OVPACK_FORMAT_VERSION:
+ raise invalid_manifest(
+ f"Unsupported ovpack format_version {version}; "
+ f"this OpenViking requires {OVPACK_FORMAT_VERSION}",
+ manifest_path,
+ format_version=version_int,
+ supported_format_version=OVPACK_FORMAT_VERSION,
+ )
+ if manifest.get("kind") != OVPACK_KIND:
+ raise invalid_manifest(
+ "Invalid ovpack manifest kind",
+ manifest_path,
+ expected=OVPACK_KIND,
+ actual=manifest.get("kind"),
+ )
+ return manifest
+
+
+def validate_manifest_root_matches_zip(manifest: dict[str, Any], base_name: str) -> None:
+ root = manifest.get("root")
+ if not isinstance(root, dict):
+ raise InvalidArgumentError("Missing ovpack manifest root")
+
+ root_name = root.get("name")
+ if not isinstance(root_name, str) or not root_name:
+ raise InvalidArgumentError(
+ "Missing ovpack manifest root name",
+ details={"field": "root.name"},
+ )
+ if root_name != base_name:
+ raise InvalidArgumentError(
+ "ovpack manifest root name does not match zip root",
+ details={"manifest_root_name": root_name, "zip_root": base_name},
+ )
+
+ root_uri = manifest_root_uri(manifest)
+ if root_uri and root_uri != "viking://" and uri_leaf_name(root_uri) != root_name:
+ raise InvalidArgumentError(
+ "ovpack manifest root name does not match root uri",
+ details={"manifest_root_name": root_name, "root_uri": root_uri},
+ )
+
+
+def manifest_entries_by_path(manifest: dict[str, Any]) -> dict[str, dict[str, Any]]:
+ if not isinstance(manifest, dict) or "entries" not in manifest:
+ return {}
+
+ entries = manifest.get("entries")
+ if not isinstance(entries, list):
+ raise InvalidArgumentError("Invalid ovpack manifest: entries must be a list")
+
+ by_path: dict[str, dict[str, Any]] = {}
+ for index, entry in enumerate(entries):
+ if not isinstance(entry, dict):
+ raise InvalidArgumentError(
+ "Invalid ovpack manifest entry",
+ details={"index": index},
+ )
+
+ rel_path = entry.get("path")
+ kind = entry.get("kind")
+ if not isinstance(rel_path, str):
+ raise InvalidArgumentError(
+ "Invalid ovpack manifest entry path",
+ details={"index": index},
+ )
+ validate_ovpack_rel_path(rel_path)
+ if kind not in {"directory", "file"}:
+ raise InvalidArgumentError(
+ "Invalid ovpack manifest entry kind",
+ details={"path": rel_path, "kind": kind},
+ )
+ if rel_path in by_path:
+ raise InvalidArgumentError(
+ "Duplicate ovpack manifest entry",
+ details={"path": rel_path},
+ )
+ by_path[rel_path] = entry
+
+ return by_path
+
+
+def manifest_index_records_info(manifest: dict[str, Any]) -> dict[str, Any]:
+ index = manifest.get("index")
+ if not isinstance(index, dict):
+ raise InvalidArgumentError("Missing ovpack index", details={"field": "index"})
+
+ records = index.get("records")
+ if not isinstance(records, dict):
+ raise InvalidArgumentError(
+ "Missing ovpack index records",
+ details={"field": "index.records"},
+ )
+ if records.get("path") != OVPACK_INDEX_RECORDS_PATH:
+ raise InvalidArgumentError(
+ "Invalid ovpack index records path",
+ details={"path": records.get("path")},
+ )
+ count = records.get("count")
+ if not isinstance(count, int) or isinstance(count, bool) or count < 0:
+ raise InvalidArgumentError(
+ "Invalid ovpack index records count",
+ details={"count": count},
+ )
+ normalize_sha256(records.get("sha256"), field="index.records.sha256")
+ return records
+
+
+def manifest_dense_info(manifest: dict[str, Any]) -> dict[str, Any] | None:
+ index = manifest.get("index")
+ if not isinstance(index, dict):
+ return None
+ dense = index.get("dense")
+ if dense is None:
+ return None
+ if not isinstance(dense, dict):
+ raise InvalidArgumentError(
+ "Invalid ovpack dense vector index",
+ details={"field": "index.dense"},
+ )
+ if dense.get("path") != OVPACK_DENSE_PATH:
+ raise InvalidArgumentError(
+ "Invalid ovpack dense vector path",
+ details={"path": dense.get("path")},
+ )
+ count = dense.get("count")
+ dimensions = dense.get("dimensions")
+ if not isinstance(count, int) or isinstance(count, bool) or count < 0:
+ raise InvalidArgumentError(
+ "Invalid ovpack dense vector count",
+ details={"count": count},
+ )
+ if not isinstance(dimensions, int) or isinstance(dimensions, bool) or dimensions <= 0:
+ raise InvalidArgumentError(
+ "Invalid ovpack dense vector dimensions",
+ details={"dimensions": dimensions},
+ )
+ if dense.get("dtype") != "float32" or dense.get("byte_order") != "little":
+ raise InvalidArgumentError(
+ "Unsupported ovpack dense vector encoding",
+ details={
+ "dtype": dense.get("dtype"),
+ "byte_order": dense.get("byte_order"),
+ },
+ )
+ normalize_sha256(dense.get("sha256"), field="index.dense.sha256")
+ return dense
diff --git a/openviking/storage/ovpack/operations.py b/openviking/storage/ovpack/operations.py
new file mode 100644
index 000000000..34c9a0d79
--- /dev/null
+++ b/openviking/storage/ovpack/operations.py
@@ -0,0 +1,668 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""OVPack export/import and backup/restore operations."""
+
+import asyncio
+import json
+import os
+import zipfile
+from typing import Any, Optional
+
+from openviking.core.namespace import context_type_for_uri, relative_uri_path
+from openviking.server.identity import RequestContext
+from openviking.storage.index_consistency import check_index_consistency
+from openviking.storage.ovpack.format import (
+ OVPACK_BACKUP_NAME,
+ OVPACK_BACKUP_TYPE,
+ OVPACK_FILES_DIR,
+ OVPACK_INDEX_RECORDS_PATH,
+ OVPACK_INTERNAL_DIR,
+ OVPACK_MANIFEST_ZIP_LEAF,
+ ensure_dir_exists,
+ ensure_ovpack_extension,
+ get_ovpack_zip_path,
+ internal_zip_path,
+ join_uri,
+ jsonl_bytes,
+ leaf_name,
+ manifest_content_sha256,
+ normalize_on_conflict,
+ normalize_vector_mode,
+ sha256_hex,
+ strip_uri_trailing_slash,
+)
+from openviking.storage.ovpack.index import build_manifest, read_text_if_exists
+from openviking.storage.ovpack.manifest import (
+ manifest_entries_by_path,
+ read_manifest,
+ validate_manifest_root_matches_zip,
+)
+from openviking.storage.ovpack.policy import (
+ NON_VECTOR_SCOPES,
+ PUBLIC_SCOPES,
+ backup_scopes_from_manifest,
+ is_backup_package,
+ is_excluded_rel_path,
+ resolve_import_root_uri,
+ validate_export_source_uri,
+ validate_import_scope_compatibility,
+ validate_import_target_uri,
+ validate_ovpack_user_rel_path,
+ validate_public_scope,
+)
+from openviking.storage.ovpack.validation import (
+ base_name_from_entries,
+ validate_manifest_content,
+ validated_import_members,
+)
+from openviking.storage.ovpack.vectors import (
+ build_dense_snapshot_manifest,
+ choose_vector_restore_action,
+ ensure_dense_snapshot_supported,
+ read_dense_vectors,
+ restore_vector_snapshot,
+)
+from openviking.utils.embedding_utils import vectorize_directory_meta, vectorize_file
+from openviking_cli.exceptions import ConflictError, InvalidArgumentError, NotFoundError
+from openviking_cli.utils.logger import get_logger
+from openviking_cli.utils.uri import VikingURI
+
+logger = get_logger(__name__)
+
+
+def _index_records_by_level(
+ index_records: list[dict[str, Any]], rel_path: str
+) -> dict[int, dict[str, Any]]:
+ by_level: dict[int, dict[str, Any]] = {}
+ for record in index_records:
+ if not isinstance(record, dict):
+ continue
+ if record.get("path") != rel_path:
+ continue
+ try:
+ level = int(record.get("level", 2))
+ except (TypeError, ValueError):
+ continue
+ by_level[level] = record
+ return by_level
+
+
+def _index_scalar_overrides(
+ index_records: list[dict[str, Any]], rel_path: str
+) -> dict[int, dict[str, Any]]:
+ overrides: dict[int, dict[str, Any]] = {}
+ for level, record in _index_records_by_level(index_records, rel_path).items():
+ scalars = record.get("scalars")
+ if isinstance(scalars, dict):
+ overrides[level] = dict(scalars)
+ return overrides
+
+
+async def _root_exists(viking_fs, root_uri: str, ctx: RequestContext) -> bool:
+ try:
+ await viking_fs.ls(root_uri, ctx=ctx)
+ return True
+ except NotFoundError:
+ return False
+ except FileNotFoundError:
+ return False
+
+
+async def _ensure_parent_exists(viking_fs, parent: str, ctx: RequestContext) -> None:
+ try:
+ await viking_fs.stat(parent, ctx=ctx)
+ except Exception:
+ await viking_fs.mkdir(parent, ctx=ctx)
+
+
+async def _remove_existing_root(viking_fs, root_uri: str, ctx: RequestContext) -> None:
+ if not hasattr(viking_fs, "rm"):
+ logger.warning(f"[ovpack] Cannot remove existing resource without rm(): {root_uri}")
+ return
+ try:
+ await viking_fs.rm(root_uri, recursive=True, ctx=ctx)
+ except NotFoundError:
+ return
+ except FileNotFoundError:
+ return
+
+
+async def _existing_scope_roots(
+ viking_fs, scopes: tuple[str, ...], ctx: RequestContext
+) -> list[str]:
+ existing: list[str] = []
+ for scope in scopes:
+ scope_uri = f"viking://{scope}"
+ if await _root_exists(viking_fs, scope_uri, ctx):
+ existing.append(scope_uri)
+ return existing
+
+
+def _exportable_entries(entries: list[dict[str, Any]]) -> list[dict[str, Any]]:
+ exportable: list[dict[str, Any]] = []
+ for entry in entries:
+ rel_path = str(entry.get("rel_path") or "")
+ validate_ovpack_user_rel_path(rel_path, operation="export")
+ if not is_excluded_rel_path(rel_path):
+ exportable.append(entry)
+ return exportable
+
+
+async def _enqueue_direct_vectorization(
+ viking_fs,
+ uri: str,
+ ctx: RequestContext,
+ index_records: Optional[list[dict[str, Any]]] = None,
+ manifest_path_root_uri: Optional[str] = None,
+) -> None:
+ index_records = index_records or []
+ manifest_path_root_uri = manifest_path_root_uri or uri
+ entries = await viking_fs.tree(uri, node_limit=None, level_limit=None, ctx=ctx)
+ dir_uris = {uri}
+ file_entries: list[tuple[str, str, str, str]] = []
+ for entry in entries:
+ entry_uri = entry.get("uri")
+ if not entry_uri:
+ continue
+ rel_path = entry.get("rel_path") or relative_uri_path(uri, entry_uri)
+ manifest_rel_path = relative_uri_path(manifest_path_root_uri, entry_uri)
+ if entry.get("isDir"):
+ dir_uris.add(entry_uri)
+ continue
+ name = entry.get("name", "") or leaf_name(rel_path)
+ if name.startswith("."):
+ continue
+ parent = VikingURI(entry_uri).parent
+ if parent:
+ file_entries.append((entry_uri, parent.uri, name, manifest_rel_path))
+
+ async def index_dir(dir_uri: str) -> None:
+ rel_path = relative_uri_path(manifest_path_root_uri, dir_uri)
+ records_by_level = _index_records_by_level(index_records, rel_path)
+ scalar_overrides = _index_scalar_overrides(index_records, rel_path)
+ abstract = str(records_by_level.get(0, {}).get("text") or "")
+ overview = str(records_by_level.get(1, {}).get("text") or "")
+
+ if not abstract:
+ abstract = await read_text_if_exists(viking_fs, f"{dir_uri}/.abstract.md", ctx)
+ if not overview:
+ overview = await read_text_if_exists(viking_fs, f"{dir_uri}/.overview.md", ctx)
+
+ if not abstract and not overview and not scalar_overrides:
+ return
+ await vectorize_directory_meta(
+ dir_uri,
+ abstract,
+ overview,
+ context_type=context_type_for_uri(dir_uri),
+ ctx=ctx,
+ include_overview=bool(overview),
+ scalar_overrides=scalar_overrides,
+ )
+
+ async def index_file(file_uri: str, parent_uri: str, name: str, rel_path: str) -> None:
+ overrides = _index_scalar_overrides(index_records, rel_path)
+ scalar_override = overrides.get(2) or next(iter(overrides.values()), {})
+ summary = str(scalar_override.get("abstract") or "")
+ await vectorize_file(
+ file_path=file_uri,
+ summary_dict={"name": name, "summary": summary},
+ parent_uri=parent_uri,
+ context_type=context_type_for_uri(file_uri),
+ ctx=ctx,
+ scalar_override=scalar_override,
+ )
+
+ await asyncio.gather(*(index_dir(dir_uri) for dir_uri in dir_uris))
+ await asyncio.gather(
+ *(
+ index_file(file_uri, parent_uri, file_name, rel_path)
+ for file_uri, parent_uri, file_name, rel_path in file_entries
+ )
+ )
+
+
+async def import_ovpack(
+ viking_fs,
+ file_path: str,
+ parent: str,
+ ctx: RequestContext,
+ on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
+ vector_store=None,
+) -> str:
+ """
+ Import .ovpack file to the specified parent path.
+
+ Args:
+ viking_fs: VikingFS instance
+ file_path: Local .ovpack file path
+ parent: Target parent URI (e.g., viking://resources/...)
+ on_conflict: One of "fail", "overwrite", or "skip"
+ vector_mode: One of "auto", "recompute", or "require"
+
+ Returns:
+ Root resource URI after import
+ """
+ if not os.path.exists(file_path):
+ raise FileNotFoundError(f"File not found: {file_path}")
+
+ parent = strip_uri_trailing_slash(parent)
+ validate_public_scope(parent, operation="import", allow_root=True)
+ conflict_action = normalize_on_conflict(on_conflict)
+ vector_action_mode = normalize_vector_mode(vector_mode)
+ index_records: list[dict[str, Any]] = []
+ dense_vectors: dict[str, list[float]] = {}
+ vector_action = "recompute"
+
+ with zipfile.ZipFile(file_path, "r") as zf:
+ infolist = zf.infolist()
+ if not infolist:
+ raise ValueError("Empty ovpack file")
+
+ base_name = base_name_from_entries(infolist)
+ manifest = read_manifest(zf, base_name)
+ validate_manifest_root_matches_zip(manifest, base_name)
+ root_uri = resolve_import_root_uri(parent, base_name, manifest)
+ validate_import_scope_compatibility(manifest, root_uri)
+ validate_import_target_uri(root_uri)
+
+ members = validated_import_members(infolist, base_name, root_uri)
+ existing_roots = [root_uri] if await _root_exists(viking_fs, root_uri, ctx) else []
+
+ if existing_roots:
+ if conflict_action == "skip":
+ logger.info(f"[ovpack] Skipped existing resource at {root_uri}")
+ return root_uri
+ if conflict_action == "fail":
+ resource = existing_roots[0]
+ raise ConflictError(
+ f"Resource already exists at {resource}. "
+ "Use on_conflict='overwrite' to replace it.",
+ resource=resource,
+ )
+
+ index_records = validate_manifest_content(zf, manifest, infolist, base_name)
+ dense_vectors = read_dense_vectors(zf, manifest, base_name, index_records)
+ if VikingURI(root_uri).scope not in NON_VECTOR_SCOPES:
+ vector_action = choose_vector_restore_action(
+ manifest,
+ index_records,
+ dense_vectors,
+ vector_store=vector_store,
+ vector_mode=vector_action_mode,
+ )
+ if parent != "viking://":
+ await _ensure_parent_exists(viking_fs, parent, ctx)
+
+ for existing_root in existing_roots:
+ logger.info(f"[ovpack] Overwriting existing resource at {existing_root}")
+ await _remove_existing_root(viking_fs, existing_root, ctx)
+
+ for _, safe_zip_path, kind, rel_path in members:
+ if kind in {"manifest", "internal"}:
+ continue
+ if kind == "directory":
+ await viking_fs.mkdir(join_uri(root_uri, rel_path), exist_ok=True, ctx=ctx)
+ continue
+
+ target_file_uri = join_uri(root_uri, rel_path)
+ data = zf.read(safe_zip_path)
+ await viking_fs.write_file_bytes(target_file_uri, data, ctx=ctx)
+
+ logger.info(f"[ovpack] Successfully imported {file_path} to {root_uri}")
+
+ if VikingURI(root_uri).scope not in NON_VECTOR_SCOPES:
+ if vector_action == "restore":
+ await restore_vector_snapshot(vector_store, root_uri, index_records, dense_vectors, ctx)
+ logger.info(f"[ovpack] Restored vector snapshot for: {root_uri}")
+ else:
+ await _enqueue_direct_vectorization(
+ viking_fs,
+ root_uri,
+ ctx=ctx,
+ index_records=index_records,
+ )
+ logger.info(f"[ovpack] Enqueued direct vectorization for: {root_uri}")
+ else:
+ logger.info(f"[ovpack] Skipped vectorization for non-vector scope: {root_uri}")
+
+ return root_uri
+
+
+async def _backup_entries(viking_fs, ctx: RequestContext) -> list[dict[str, Any]]:
+ entries: list[dict[str, Any]] = []
+ for scope in PUBLIC_SCOPES:
+ scope_uri = f"viking://{scope}"
+ entries.append(
+ {
+ "rel_path": scope,
+ "uri": scope_uri,
+ "isDir": True,
+ "size": 0,
+ }
+ )
+ try:
+ scope_entries = await viking_fs.tree(
+ scope_uri,
+ show_all_hidden=True,
+ node_limit=None,
+ level_limit=None,
+ ctx=ctx,
+ )
+ except (NotFoundError, FileNotFoundError):
+ continue
+
+ for entry in _exportable_entries(scope_entries):
+ rel_path = entry.get("rel_path", "")
+ if not rel_path:
+ continue
+ scoped_entry = dict(entry)
+ scoped_entry["rel_path"] = f"{scope}/{rel_path}"
+ scoped_entry["uri"] = join_uri(scope_uri, rel_path)
+ entries.append(scoped_entry)
+ return entries
+
+
+async def _write_ovpack_archive(
+ viking_fs,
+ root_uri: str,
+ to: str,
+ base_name: str,
+ entries: list[dict[str, Any]],
+ manifest: dict[str, Any],
+ index_records: list[dict[str, Any]],
+ dense_values: list[float],
+ ctx: RequestContext,
+) -> str:
+ ensure_dir_exists(to)
+ manifest_entries = manifest_entries_by_path(manifest)
+ manifest_file_entries = {
+ rel_path: entry
+ for rel_path, entry in manifest_entries.items()
+ if entry.get("kind") == "file"
+ }
+
+ with zipfile.ZipFile(to, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as zf:
+ zf.writestr(base_name + "/", "")
+ zf.writestr(f"{base_name}/{OVPACK_FILES_DIR}/", "")
+
+ content_entries = [entry for entry in entries if entry["rel_path"] != ""]
+ for entry in content_entries:
+ rel_path = entry["rel_path"]
+ zip_path = get_ovpack_zip_path(base_name, rel_path)
+
+ if entry.get("isDir"):
+ zf.writestr(zip_path.rstrip("/") + "/", "")
+ else:
+ full_uri = entry.get("uri") or join_uri(root_uri, rel_path)
+ try:
+ data = await viking_fs.read_file_bytes(full_uri, ctx=ctx)
+ except Exception as exc:
+ logger.warning(f"Failed to export file {full_uri}: {exc}")
+ raise
+
+ manifest_entry = manifest_file_entries.get(rel_path)
+ if manifest_entry is not None:
+ manifest_entry["size"] = len(data)
+ manifest_entry["sha256"] = sha256_hex(data)
+ zf.writestr(zip_path, data)
+
+ manifest["content_sha256"] = manifest_content_sha256(manifest_file_entries)
+ index_bytes = jsonl_bytes(index_records)
+ manifest["index"] = {
+ "records": {
+ "path": OVPACK_INDEX_RECORDS_PATH,
+ "count": len(index_records),
+ "sha256": sha256_hex(index_bytes),
+ }
+ }
+
+ zf.writestr(f"{base_name}/{OVPACK_INTERNAL_DIR}/", "")
+ zf.writestr(internal_zip_path(base_name, OVPACK_INDEX_RECORDS_PATH), index_bytes)
+
+ dense_snapshot = build_dense_snapshot_manifest(index_records, dense_values)
+ if dense_snapshot is not None:
+ dense_bytes, dense_manifest = dense_snapshot
+ manifest["index"]["dense"] = dense_manifest
+ zf.writestr(internal_zip_path(base_name, dense_manifest["path"]), dense_bytes)
+
+ zf.writestr(
+ f"{base_name}/{OVPACK_MANIFEST_ZIP_LEAF}",
+ json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2).encode("utf-8"),
+ )
+ return to
+
+
+async def export_ovpack(
+ viking_fs,
+ uri: str,
+ to: str,
+ ctx: RequestContext,
+ vector_store=None,
+ include_vectors: bool = False,
+) -> str:
+ """
+ Export the specified context path as a .ovpack file.
+
+ Args:
+ viking_fs: VikingFS instance
+ uri: Viking URI
+ to: Target file path (can be an existing directory or a path ending with .ovpack)
+ vector_store: Optional vector store used to export portable index metadata
+ include_vectors: Whether to include pure-dense vector snapshots
+
+ Returns:
+ Exported file path
+
+ """
+ uri = strip_uri_trailing_slash(uri)
+ validate_export_source_uri(uri)
+
+ base_name = leaf_name(uri) or "export"
+
+ if os.path.isdir(to):
+ to = os.path.join(to, f"{base_name}.ovpack")
+ else:
+ to = ensure_ovpack_extension(to)
+
+ entries = _exportable_entries(
+ await viking_fs.tree(
+ uri,
+ show_all_hidden=True,
+ node_limit=None,
+ level_limit=None,
+ ctx=ctx,
+ )
+ )
+ if include_vectors:
+ ensure_dense_snapshot_supported(vector_store)
+ report = await check_index_consistency(
+ viking_fs,
+ vector_store,
+ uri,
+ entries,
+ ctx,
+ )
+ if not report.ok:
+ raise InvalidArgumentError(
+ "Cannot export incomplete OpenViking vector index snapshot",
+ details=report.details(),
+ )
+ manifest, index_records, dense_values = await build_manifest(
+ viking_fs,
+ vector_store,
+ uri,
+ base_name,
+ entries,
+ ctx,
+ include_vectors=include_vectors,
+ )
+ await _write_ovpack_archive(
+ viking_fs,
+ uri,
+ to,
+ base_name,
+ entries,
+ manifest,
+ index_records,
+ dense_values,
+ ctx,
+ )
+
+ logger.info(f"[ovpack] Exported {uri} to {to}")
+ return to
+
+
+async def backup_ovpack(
+ viking_fs,
+ to: str,
+ ctx: RequestContext,
+ vector_store=None,
+ include_vectors: bool = False,
+) -> str:
+ """Export all public OpenViking scopes as a restore-only backup package."""
+ base_name = OVPACK_BACKUP_NAME
+ if os.path.isdir(to):
+ to = os.path.join(to, f"{base_name}.ovpack")
+ else:
+ to = ensure_ovpack_extension(to)
+
+ entries = await _backup_entries(viking_fs, ctx)
+ if include_vectors:
+ ensure_dense_snapshot_supported(vector_store)
+ report = await check_index_consistency(
+ viking_fs,
+ vector_store,
+ "viking://",
+ entries,
+ ctx,
+ )
+ if not report.ok:
+ raise InvalidArgumentError(
+ "Cannot export incomplete OpenViking vector index snapshot",
+ details=report.details(),
+ )
+ manifest, index_records, dense_values = await build_manifest(
+ viking_fs,
+ vector_store,
+ "viking://",
+ base_name,
+ entries,
+ ctx,
+ package_type=OVPACK_BACKUP_TYPE,
+ scopes=list(PUBLIC_SCOPES),
+ include_vectors=include_vectors,
+ )
+ await _write_ovpack_archive(
+ viking_fs,
+ "viking://",
+ to,
+ base_name,
+ entries,
+ manifest,
+ index_records,
+ dense_values,
+ ctx,
+ )
+
+ logger.info(f"[ovpack] Backed up OpenViking public scopes to {to}")
+ return to
+
+
+async def restore_ovpack(
+ viking_fs,
+ file_path: str,
+ ctx: RequestContext,
+ on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
+ vector_store=None,
+) -> str:
+ """Restore a backup package to its original public scope roots."""
+ if not os.path.exists(file_path):
+ raise FileNotFoundError(f"File not found: {file_path}")
+
+ conflict_action = normalize_on_conflict(on_conflict)
+ vector_action_mode = normalize_vector_mode(vector_mode)
+ root_uri = "viking://"
+ index_records: list[dict[str, Any]] = []
+ dense_vectors: dict[str, list[float]] = {}
+ vector_action = "recompute"
+
+ with zipfile.ZipFile(file_path, "r") as zf:
+ infolist = zf.infolist()
+ if not infolist:
+ raise ValueError("Empty ovpack file")
+
+ base_name = base_name_from_entries(infolist)
+ manifest = read_manifest(zf, base_name)
+ validate_manifest_root_matches_zip(manifest, base_name)
+ if not is_backup_package(manifest):
+ raise InvalidArgumentError(
+ "Only backup ovpack packages can be restored with ov restore or the restore API",
+ details={"root": base_name},
+ )
+
+ manifest_entries = manifest_entries_by_path(manifest)
+ backup_scopes = backup_scopes_from_manifest(manifest, manifest_entries)
+ members = validated_import_members(infolist, base_name, root_uri)
+ existing_roots = await _existing_scope_roots(viking_fs, backup_scopes, ctx)
+
+ if existing_roots:
+ if conflict_action == "skip":
+ logger.info("[ovpack] Skipped backup restore because target scopes exist")
+ return root_uri
+ if conflict_action == "fail":
+ resource = existing_roots[0]
+ raise ConflictError(
+ f"Resource already exists at {resource}. "
+ "Use on_conflict='overwrite' to replace it.",
+ resource=resource,
+ )
+
+ index_records = validate_manifest_content(zf, manifest, infolist, base_name)
+ dense_vectors = read_dense_vectors(zf, manifest, base_name, index_records)
+ vector_action = choose_vector_restore_action(
+ manifest,
+ index_records,
+ dense_vectors,
+ vector_store=vector_store,
+ vector_mode=vector_action_mode,
+ )
+
+ for existing_root in existing_roots:
+ logger.info(f"[ovpack] Overwriting existing resource at {existing_root}")
+ await _remove_existing_root(viking_fs, existing_root, ctx)
+
+ for _, safe_zip_path, kind, rel_path in members:
+ if kind in {"manifest", "internal"} or rel_path == "":
+ continue
+ if kind == "directory":
+ await viking_fs.mkdir(join_uri(root_uri, rel_path), exist_ok=True, ctx=ctx)
+ continue
+
+ data = zf.read(safe_zip_path)
+ await viking_fs.write_file_bytes(join_uri(root_uri, rel_path), data, ctx=ctx)
+
+ logger.info(f"[ovpack] Successfully restored backup {file_path}")
+
+ if vector_action == "restore":
+ await restore_vector_snapshot(vector_store, root_uri, index_records, dense_vectors, ctx)
+ logger.info("[ovpack] Restored vector snapshot for backup")
+ return root_uri
+
+ for scope in backup_scopes:
+ if scope in NON_VECTOR_SCOPES:
+ logger.info(f"[ovpack] Skipped vectorization for non-vector scope: {scope}")
+ continue
+ scope_uri = f"viking://{scope}"
+ await _enqueue_direct_vectorization(
+ viking_fs,
+ scope_uri,
+ ctx=ctx,
+ index_records=index_records,
+ manifest_path_root_uri=root_uri,
+ )
+ logger.info(f"[ovpack] Enqueued direct vectorization for: {scope_uri}")
+
+ return root_uri
diff --git a/openviking/storage/ovpack/policy.py b/openviking/storage/ovpack/policy.py
new file mode 100644
index 000000000..4561c1b73
--- /dev/null
+++ b/openviking/storage/ovpack/policy.py
@@ -0,0 +1,215 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""OVPack URI scope and import/restore policy helpers."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from openviking.core.namespace import uri_depth, uri_parts
+from openviking.resource.watch_storage import is_watch_task_control_uri
+from openviking.storage.ovpack.format import (
+ OVPACK_BACKUP_TYPE,
+ join_uri,
+ leaf_name,
+ strip_uri_trailing_slash,
+ validate_ovpack_rel_path,
+)
+from openviking_cli.exceptions import InvalidArgumentError
+from openviking_cli.utils.uri import VikingURI
+
+PUBLIC_SCOPES = ("resources", "user", "agent", "session")
+IMPORTABLE_SCOPES = frozenset(PUBLIC_SCOPES)
+STRUCTURED_IMPORT_SCOPES = frozenset({"user", "agent", "session"})
+EXCLUDED_FILENAMES = frozenset({".relations.json"})
+NON_VECTOR_SCOPES = frozenset({"session"})
+
+
+def is_excluded_rel_path(rel_path: str) -> bool:
+ return leaf_name(rel_path) in EXCLUDED_FILENAMES
+
+
+def validate_ovpack_user_rel_path(rel_path: str, *, operation: str) -> None:
+ try:
+ validate_ovpack_rel_path(rel_path)
+ except InvalidArgumentError as exc:
+ raise InvalidArgumentError(
+ f"cannot {operation} unsafe ovpack path",
+ details={"path": rel_path},
+ ) from exc
+
+
+def _scope_relative_path(uri: str) -> str:
+ parts = uri_parts(uri)
+ if len(parts) <= 1:
+ return ""
+ return "/".join(parts[1:])
+
+
+def validate_public_scope(uri: str, *, operation: str, allow_root: bool = False) -> None:
+ parsed = VikingURI(uri)
+ if parsed.uri == "viking://":
+ if allow_root:
+ return
+ raise InvalidArgumentError(f"ovpack {operation} is not supported for root URI")
+ if parsed.scope not in IMPORTABLE_SCOPES:
+ raise InvalidArgumentError(f"ovpack {operation} is not supported for scope: {parsed.scope}")
+
+
+def validate_import_target_uri(uri: str) -> None:
+ """Enforce the same target-policy boundary as direct content writes."""
+ validate_public_scope(uri, operation="import")
+ validate_ovpack_user_rel_path(_scope_relative_path(uri), operation="import")
+ name = leaf_name(uri)
+ if name in EXCLUDED_FILENAMES:
+ raise InvalidArgumentError(f"cannot import internal ovpack file: {uri}")
+ if is_watch_task_control_uri(uri):
+ raise InvalidArgumentError(f"cannot import watch task control file: {uri}")
+
+
+def validate_export_source_uri(uri: str) -> None:
+ validate_public_scope(uri, operation="export")
+ validate_ovpack_user_rel_path(_scope_relative_path(uri), operation="export")
+ name = leaf_name(uri)
+ if name in EXCLUDED_FILENAMES:
+ raise InvalidArgumentError(f"cannot export internal ovpack file: {uri}")
+ if is_watch_task_control_uri(uri):
+ raise InvalidArgumentError(f"cannot export watch task control file: {uri}")
+
+
+def manifest_root_uri(manifest: dict[str, Any]) -> str:
+ root = manifest.get("root")
+ if not isinstance(root, dict):
+ return ""
+ uri = root.get("uri")
+ if isinstance(uri, str):
+ try:
+ return strip_uri_trailing_slash(uri)
+ except Exception:
+ return uri.rstrip("/")
+ return ""
+
+
+def is_backup_package(manifest: dict[str, Any]) -> bool:
+ root = manifest.get("root")
+ return (
+ isinstance(root, dict)
+ and root.get("package_type") == OVPACK_BACKUP_TYPE
+ and manifest_root_uri(manifest) == "viking://"
+ )
+
+
+def is_top_level_scope_package(manifest: dict[str, Any]) -> bool:
+ return manifest_root_uri(manifest) in {f"viking://{scope}" for scope in IMPORTABLE_SCOPES}
+
+
+def resolve_import_root_uri(parent: str, base_name: str, manifest: dict[str, Any]) -> str:
+ if is_backup_package(manifest):
+ raise InvalidArgumentError(
+ "Backup ovpack packages must be restored with ov restore or the restore API",
+ details={"root": base_name, "parent": parent},
+ )
+
+ if parent == "viking://":
+ if not is_top_level_scope_package(manifest):
+ raise InvalidArgumentError(
+ "Only top-level scope ovpack packages can be imported to viking://",
+ details={"root": base_name},
+ )
+ return manifest_root_uri(manifest)
+
+ if is_top_level_scope_package(manifest):
+ raise InvalidArgumentError(
+ "Top-level scope ovpack packages must be imported to viking://",
+ details={"root": base_name, "parent": parent},
+ )
+ return join_uri(parent, base_name)
+
+
+def _parse_import_uri(uri: str, *, field: str) -> VikingURI:
+ if not uri:
+ raise InvalidArgumentError(f"Missing ovpack {field}")
+ try:
+ return VikingURI(uri)
+ except ValueError as exc:
+ raise InvalidArgumentError(f"Invalid ovpack {field}", details={field: uri}) from exc
+
+
+def validate_import_scope_compatibility(manifest: dict[str, Any], target_root_uri: str) -> None:
+ source_root_uri = manifest_root_uri(manifest)
+ source = _parse_import_uri(source_root_uri, field="manifest root uri")
+ target = _parse_import_uri(target_root_uri, field="target root uri")
+
+ if source.scope not in IMPORTABLE_SCOPES:
+ raise InvalidArgumentError(
+ "ovpack import is not supported for source scope",
+ details={"source_scope": source.scope},
+ )
+ if source.scope != target.scope:
+ raise InvalidArgumentError(
+ "ovpack source scope does not match target scope",
+ details={
+ "source_scope": source.scope,
+ "target_scope": target.scope,
+ },
+ )
+ if source.scope in STRUCTURED_IMPORT_SCOPES and uri_depth(source.uri) != uri_depth(target.uri):
+ raise InvalidArgumentError(
+ "ovpack source path is incompatible with target path",
+ details={"source": source_root_uri, "target": target_root_uri},
+ )
+
+
+def backup_scopes_from_manifest(
+ manifest: dict[str, Any], manifest_entries: dict[str, dict[str, Any]]
+) -> tuple[str, ...]:
+ roots = {rel_path.split("/", 1)[0] for rel_path in manifest_entries if rel_path}
+ unexpected = sorted(root for root in roots if root not in IMPORTABLE_SCOPES)
+ if unexpected:
+ raise InvalidArgumentError(
+ "Backup ovpack contains unsupported roots",
+ details={"roots": unexpected},
+ )
+
+ directory_scope_roots = {
+ rel_path
+ for rel_path, entry in manifest_entries.items()
+ if rel_path in IMPORTABLE_SCOPES and entry.get("kind") == "directory"
+ }
+ missing_scope_directories = sorted(roots - directory_scope_roots)
+ if missing_scope_directories:
+ raise InvalidArgumentError(
+ "Backup ovpack scope roots must be directory entries",
+ details={"missing_scope_directories": missing_scope_directories},
+ )
+
+ entry_scopes = tuple(scope for scope in PUBLIC_SCOPES if scope in directory_scope_roots)
+ declared_scopes = manifest.get("scopes")
+ if not isinstance(declared_scopes, list) or any(
+ not isinstance(scope, str) for scope in declared_scopes
+ ):
+ raise InvalidArgumentError(
+ "Invalid backup ovpack scopes",
+ details={"field": "scopes"},
+ )
+ duplicate_scopes = sorted(
+ scope for scope in set(declared_scopes) if declared_scopes.count(scope) > 1
+ )
+ invalid_scopes = sorted(scope for scope in declared_scopes if scope not in IMPORTABLE_SCOPES)
+ if duplicate_scopes or invalid_scopes:
+ raise InvalidArgumentError(
+ "Invalid backup ovpack scopes",
+ details={
+ "duplicate_scopes": duplicate_scopes,
+ "invalid_scopes": invalid_scopes,
+ },
+ )
+ if set(declared_scopes) != set(entry_scopes):
+ raise InvalidArgumentError(
+ "Backup ovpack scopes do not match entries",
+ details={
+ "declared_scopes": declared_scopes,
+ "entry_scopes": list(entry_scopes),
+ },
+ )
+ return entry_scopes
diff --git a/openviking/storage/ovpack/validation.py b/openviking/storage/ovpack/validation.py
new file mode 100644
index 000000000..53faecf4b
--- /dev/null
+++ b/openviking/storage/ovpack/validation.py
@@ -0,0 +1,512 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""OVPack archive/member validation helpers."""
+
+from __future__ import annotations
+
+import json
+import zipfile
+from typing import Any
+
+from openviking.storage.ovpack.format import (
+ OVPACK_DENSE_PATH,
+ OVPACK_INDEX_RECORDS_PATH,
+ OVPACK_MANIFEST_ZIP_LEAF,
+ get_viking_rel_path_from_zip,
+ internal_zip_path,
+ is_content_zip_path,
+ is_internal_zip_path,
+ is_manifest_zip_path,
+ join_uri,
+ manifest_content_sha256,
+ normalize_sha256,
+ sha256_hex,
+ validate_ovpack_member_path,
+)
+from openviking.storage.ovpack.manifest import (
+ manifest_dense_info,
+ manifest_entries_by_path,
+ manifest_index_records_info,
+)
+from openviking.storage.ovpack.policy import validate_import_target_uri
+from openviking_cli.exceptions import InvalidArgumentError
+
+
+def base_name_from_entries(infolist: list[zipfile.ZipInfo]) -> str:
+ for info in infolist:
+ filename = info.filename
+ if filename:
+ base_name = filename.replace("\\", "/").split("/")[0]
+ if base_name:
+ return base_name
+ raise ValueError("Could not determine root directory name from ovpack")
+
+
+def record_dense_ref(record: dict[str, Any]) -> dict[str, int] | None:
+ vector = record.get("vector")
+ if not isinstance(vector, dict):
+ return None
+ dense = vector.get("dense")
+ if not isinstance(dense, dict):
+ return None
+ offset = dense.get("offset")
+ dimensions = dense.get("dimensions")
+ if not isinstance(offset, int) or not isinstance(dimensions, int):
+ return None
+ return {"offset": offset, "dimensions": dimensions}
+
+
+def dense_record_count(index_records: list[dict[str, Any]]) -> int:
+ return sum(1 for record in index_records if record_dense_ref(record) is not None)
+
+
+def _zip_file_members_by_path(
+ infolist: list[zipfile.ZipInfo], base_name: str
+) -> dict[str, tuple[zipfile.ZipInfo, str]]:
+ files: dict[str, tuple[zipfile.ZipInfo, str]] = {}
+ for info in infolist:
+ zip_path = info.filename
+ if not zip_path:
+ continue
+ safe_zip_path = validate_ovpack_member_path(zip_path, base_name)
+ if (
+ is_manifest_zip_path(safe_zip_path, base_name)
+ or is_internal_zip_path(safe_zip_path, base_name)
+ or safe_zip_path.endswith("/")
+ ):
+ continue
+ if not is_content_zip_path(safe_zip_path, base_name):
+ raise InvalidArgumentError(
+ "Unexpected ovpack member outside files directory",
+ details={"path": safe_zip_path},
+ )
+
+ rel_path = get_viking_rel_path_from_zip(safe_zip_path)
+ if rel_path in files:
+ raise InvalidArgumentError(
+ "Duplicate ovpack file entry",
+ details={"path": rel_path},
+ )
+ files[rel_path] = (info, safe_zip_path)
+ return files
+
+
+def _zip_directory_members_by_path(infolist: list[zipfile.ZipInfo], base_name: str) -> set[str]:
+ directories: set[str] = set()
+ for info in infolist:
+ zip_path = info.filename
+ if not zip_path:
+ continue
+ safe_zip_path = validate_ovpack_member_path(zip_path, base_name)
+ stripped_zip_path = safe_zip_path.rstrip("/")
+ if is_internal_zip_path(stripped_zip_path, base_name):
+ continue
+ if stripped_zip_path == base_name:
+ continue
+ if not safe_zip_path.endswith("/"):
+ continue
+ if not is_content_zip_path(stripped_zip_path, base_name):
+ raise InvalidArgumentError(
+ "Unexpected ovpack directory outside files directory",
+ details={"path": safe_zip_path},
+ )
+
+ rel_path = get_viking_rel_path_from_zip(stripped_zip_path)
+ if rel_path in directories:
+ raise InvalidArgumentError(
+ "Duplicate ovpack directory entry",
+ details={"path": rel_path},
+ )
+ directories.add(rel_path)
+ return directories
+
+
+def _parse_index_records(raw: bytes, expected_count: int, path: str) -> list[dict[str, Any]]:
+ try:
+ text = raw.decode("utf-8")
+ except UnicodeDecodeError as exc:
+ raise InvalidArgumentError("Invalid ovpack index records encoding") from exc
+
+ lines = text.splitlines() if text else []
+ if len(lines) != expected_count:
+ raise InvalidArgumentError(
+ "ovpack index record count does not match manifest",
+ details={"path": path, "expected": expected_count, "actual": len(lines)},
+ )
+
+ records: list[dict[str, Any]] = []
+ for index, line in enumerate(lines):
+ try:
+ record = json.loads(line)
+ except json.JSONDecodeError as exc:
+ raise InvalidArgumentError(
+ "Invalid JSON in ovpack index record",
+ details={"path": path, "line": index + 1},
+ ) from exc
+ if not isinstance(record, dict):
+ raise InvalidArgumentError(
+ "Invalid ovpack index record",
+ details={"path": path, "line": index + 1},
+ )
+ records.append(record)
+ return records
+
+
+def _validate_index_record(
+ record: dict[str, Any],
+ manifest_entries: dict[str, dict[str, Any]],
+ index: int,
+) -> None:
+ record_id = record.get("record_id")
+ rel_path = record.get("path")
+ kind = record.get("kind")
+ level = record.get("level")
+ if not isinstance(record_id, str) or not record_id:
+ raise InvalidArgumentError(
+ "Invalid ovpack index record id",
+ details={"index": index},
+ )
+ if not isinstance(rel_path, str) or rel_path not in manifest_entries:
+ raise InvalidArgumentError(
+ "Invalid ovpack index record path",
+ details={"index": index, "path": rel_path},
+ )
+ if kind != manifest_entries[rel_path].get("kind"):
+ raise InvalidArgumentError(
+ "ovpack index record kind does not match manifest",
+ details={"index": index, "path": rel_path, "kind": kind},
+ )
+ if not isinstance(level, int) or isinstance(level, bool):
+ raise InvalidArgumentError(
+ "Invalid ovpack index record level",
+ details={"index": index, "path": rel_path, "level": level},
+ )
+
+ text = record.get("text")
+ if text is not None and not isinstance(text, str):
+ raise InvalidArgumentError(
+ "Invalid ovpack index record text",
+ details={"index": index, "path": rel_path},
+ )
+ scalars = record.get("scalars")
+ if scalars is not None and not isinstance(scalars, dict):
+ raise InvalidArgumentError(
+ "Invalid ovpack index record scalars",
+ details={"index": index, "path": rel_path},
+ )
+
+
+def _validate_dense_references(
+ dense_info: dict[str, Any] | None,
+ dense_data: bytes | None,
+ index_records: list[dict[str, Any]],
+) -> None:
+ refs: list[tuple[int, int]] = []
+ for index, record in enumerate(index_records):
+ vector = record.get("vector")
+ if vector is None:
+ continue
+ if not isinstance(vector, dict):
+ raise InvalidArgumentError(
+ "Invalid ovpack index record vector",
+ details={"index": index, "path": record.get("path")},
+ )
+ dense = vector.get("dense")
+ if not isinstance(dense, dict):
+ raise InvalidArgumentError(
+ "Invalid ovpack index record dense vector reference",
+ details={"index": index, "path": record.get("path")},
+ )
+ offset = dense.get("offset")
+ dimensions = dense.get("dimensions")
+ if not isinstance(offset, int) or isinstance(offset, bool) or offset < 0:
+ raise InvalidArgumentError(
+ "Invalid ovpack dense vector offset",
+ details={"index": index, "path": record.get("path"), "offset": offset},
+ )
+ if not isinstance(dimensions, int) or isinstance(dimensions, bool) or dimensions <= 0:
+ raise InvalidArgumentError(
+ "Invalid ovpack dense vector dimensions",
+ details={"index": index, "path": record.get("path"), "dimensions": dimensions},
+ )
+ refs.append((offset, dimensions))
+
+ if not refs:
+ if dense_info is not None:
+ raise InvalidArgumentError("ovpack dense vector file has no record references")
+ return
+ if dense_info is None or dense_data is None:
+ raise InvalidArgumentError(
+ "ovpack index references dense vectors but dense data is missing"
+ )
+
+ refs.sort()
+ expected_offset = 0
+ for offset, dimensions in refs:
+ if offset != expected_offset:
+ raise InvalidArgumentError(
+ "ovpack dense vector offsets are not contiguous",
+ details={"expected": expected_offset, "actual": offset},
+ )
+ if dimensions != dense_info["dimensions"]:
+ raise InvalidArgumentError(
+ "ovpack dense vector dimensions do not match manifest",
+ details={"expected": dense_info["dimensions"], "actual": dimensions},
+ )
+ expected_offset += dimensions
+
+ if dense_info["count"] != len(refs):
+ raise InvalidArgumentError(
+ "ovpack dense vector count does not match index records",
+ details={"expected": dense_info["count"], "actual": len(refs)},
+ )
+ expected_size = expected_offset * 4
+ if len(dense_data) != expected_size:
+ raise InvalidArgumentError(
+ "ovpack dense vector byte size does not match manifest",
+ details={"expected": expected_size, "actual": len(dense_data)},
+ )
+
+
+def _validate_index_content(
+ zf: zipfile.ZipFile,
+ manifest: dict[str, Any],
+ base_name: str,
+ manifest_entries: dict[str, dict[str, Any]],
+) -> list[dict[str, Any]]:
+ records_info = manifest_index_records_info(manifest)
+ records_zip_path = internal_zip_path(base_name, OVPACK_INDEX_RECORDS_PATH)
+ try:
+ records_data = zf.read(records_zip_path)
+ except KeyError as exc:
+ raise InvalidArgumentError(
+ "Missing ovpack index records",
+ details={"path": OVPACK_INDEX_RECORDS_PATH},
+ ) from exc
+ expected_records_sha = normalize_sha256(
+ records_info.get("sha256"),
+ field="index.records.sha256",
+ )
+ actual_records_sha = sha256_hex(records_data)
+ if actual_records_sha != expected_records_sha:
+ raise InvalidArgumentError(
+ "ovpack index records sha256 mismatch",
+ details={"expected": expected_records_sha, "actual": actual_records_sha},
+ )
+
+ index_records = _parse_index_records(
+ records_data,
+ records_info["count"],
+ OVPACK_INDEX_RECORDS_PATH,
+ )
+ seen_record_ids: set[str] = set()
+ for index, record in enumerate(index_records):
+ _validate_index_record(record, manifest_entries, index)
+ record_id = record["record_id"]
+ if record_id in seen_record_ids:
+ raise InvalidArgumentError(
+ "Duplicate ovpack index record id",
+ details={"record_id": record_id},
+ )
+ seen_record_ids.add(record_id)
+
+ dense_info = manifest_dense_info(manifest)
+ dense_data = None
+ if dense_info is not None:
+ dense_zip_path = internal_zip_path(base_name, OVPACK_DENSE_PATH)
+ try:
+ dense_data = zf.read(dense_zip_path)
+ except KeyError as exc:
+ raise InvalidArgumentError(
+ "Missing ovpack dense vector data",
+ details={"path": OVPACK_DENSE_PATH},
+ ) from exc
+ expected_dense_sha = normalize_sha256(
+ dense_info.get("sha256"),
+ field="index.dense.sha256",
+ )
+ actual_dense_sha = sha256_hex(dense_data)
+ if actual_dense_sha != expected_dense_sha:
+ raise InvalidArgumentError(
+ "ovpack dense vector sha256 mismatch",
+ details={"expected": expected_dense_sha, "actual": actual_dense_sha},
+ )
+
+ _validate_dense_references(dense_info, dense_data, index_records)
+ return index_records
+
+
+def _validate_internal_members(
+ infolist: list[zipfile.ZipInfo],
+ base_name: str,
+ manifest: dict[str, Any],
+) -> None:
+ expected_files = {
+ internal_zip_path(base_name, OVPACK_MANIFEST_ZIP_LEAF),
+ internal_zip_path(base_name, OVPACK_INDEX_RECORDS_PATH),
+ }
+ if manifest_dense_info(manifest) is not None:
+ expected_files.add(internal_zip_path(base_name, OVPACK_DENSE_PATH))
+
+ actual_files: set[str] = set()
+ for info in infolist:
+ safe_zip_path = validate_ovpack_member_path(info.filename, base_name)
+ if not is_internal_zip_path(safe_zip_path, base_name) or safe_zip_path.endswith("/"):
+ continue
+ actual_files.add(safe_zip_path)
+
+ missing = sorted(expected_files - actual_files)
+ unexpected = sorted(actual_files - expected_files)
+ if missing or unexpected:
+ raise InvalidArgumentError(
+ "ovpack internal entries do not match manifest",
+ details={
+ "missing_files": [path.split("/", 1)[1] for path in missing],
+ "unexpected_files": [path.split("/", 1)[1] for path in unexpected],
+ },
+ )
+
+
+def validate_manifest_content(
+ zf: zipfile.ZipFile,
+ manifest: dict[str, Any],
+ infolist: list[zipfile.ZipInfo],
+ base_name: str,
+) -> list[dict[str, Any]]:
+ if "entries" not in manifest:
+ raise InvalidArgumentError(
+ "Missing ovpack manifest entries",
+ details={"field": "entries"},
+ )
+
+ manifest_entries = manifest_entries_by_path(manifest)
+ manifest_files = {
+ rel_path: entry
+ for rel_path, entry in manifest_entries.items()
+ if entry.get("kind") == "file"
+ }
+ manifest_directories = {
+ rel_path for rel_path, entry in manifest_entries.items() if entry.get("kind") == "directory"
+ }
+ zip_files = _zip_file_members_by_path(infolist, base_name)
+ zip_directories = _zip_directory_members_by_path(infolist, base_name)
+ _validate_internal_members(infolist, base_name, manifest)
+
+ missing_files = sorted(set(manifest_files) - set(zip_files))
+ unexpected_files = sorted(set(zip_files) - set(manifest_files))
+ missing_directories = sorted(manifest_directories - zip_directories)
+ unexpected_directories = sorted(zip_directories - manifest_directories)
+ if missing_files or unexpected_files or missing_directories or unexpected_directories:
+ raise InvalidArgumentError(
+ "ovpack entries do not match manifest",
+ details={
+ "missing_files": missing_files,
+ "unexpected_files": unexpected_files,
+ "missing_directories": missing_directories,
+ "unexpected_directories": unexpected_directories,
+ },
+ )
+
+ expected_content_sha256 = manifest.get("content_sha256")
+ if expected_content_sha256 is None:
+ raise InvalidArgumentError(
+ "Missing ovpack manifest content_sha256",
+ details={"field": "content_sha256"},
+ )
+ expected_content_sha256 = normalize_sha256(
+ expected_content_sha256,
+ field="content_sha256",
+ )
+ actual_content_sha256 = manifest_content_sha256(manifest_files)
+ if actual_content_sha256 != expected_content_sha256:
+ raise InvalidArgumentError(
+ "ovpack manifest content_sha256 mismatch",
+ details={
+ "expected": expected_content_sha256,
+ "actual": actual_content_sha256,
+ },
+ )
+
+ for rel_path, (_, safe_zip_path) in sorted(zip_files.items()):
+ entry = manifest_files[rel_path]
+ data = zf.read(safe_zip_path)
+
+ expected_size = entry.get("size")
+ if expected_size is not None:
+ if (
+ not isinstance(expected_size, int)
+ or isinstance(expected_size, bool)
+ or expected_size < 0
+ ):
+ raise InvalidArgumentError(
+ "Invalid ovpack manifest file size",
+ details={"path": rel_path, "size": expected_size},
+ )
+ if len(data) != expected_size:
+ raise InvalidArgumentError(
+ "ovpack file size does not match manifest",
+ details={
+ "path": rel_path,
+ "expected": expected_size,
+ "actual": len(data),
+ },
+ )
+
+ expected_sha256 = entry.get("sha256")
+ if expected_sha256 is not None:
+ expected_sha256 = normalize_sha256(
+ expected_sha256,
+ field="sha256",
+ path=rel_path,
+ )
+ actual_sha256 = sha256_hex(data)
+ if actual_sha256 != expected_sha256:
+ raise InvalidArgumentError(
+ "ovpack file sha256 does not match manifest",
+ details={
+ "path": rel_path,
+ "expected": expected_sha256,
+ "actual": actual_sha256,
+ },
+ )
+
+ return _validate_index_content(zf, manifest, base_name, manifest_entries)
+
+
+def validated_import_members(
+ infolist: list[zipfile.ZipInfo], base_name: str, root_uri: str
+) -> list[tuple[zipfile.ZipInfo, str, str, str]]:
+ members: list[tuple[zipfile.ZipInfo, str, str, str]] = []
+ for info in infolist:
+ zip_path = info.filename
+ if not zip_path:
+ continue
+
+ safe_zip_path = validate_ovpack_member_path(zip_path, base_name)
+ stripped_zip_path = safe_zip_path.rstrip("/")
+ if is_manifest_zip_path(safe_zip_path, base_name):
+ members.append((info, safe_zip_path, "manifest", ""))
+ continue
+ if is_internal_zip_path(stripped_zip_path, base_name):
+ members.append((info, safe_zip_path, "internal", ""))
+ continue
+ if stripped_zip_path == base_name:
+ continue
+ if not is_content_zip_path(stripped_zip_path, base_name):
+ raise InvalidArgumentError(
+ "Unexpected ovpack member outside files directory",
+ details={"path": safe_zip_path},
+ )
+
+ kind = "directory" if safe_zip_path.endswith("/") else "file"
+ rel_path = get_viking_rel_path_from_zip(
+ safe_zip_path.rstrip("/") if kind == "directory" else safe_zip_path
+ )
+ if root_uri == "viking://" and rel_path == "":
+ members.append((info, safe_zip_path, kind, rel_path))
+ continue
+ target_uri = join_uri(root_uri, rel_path)
+ validate_import_target_uri(target_uri)
+ members.append((info, safe_zip_path, kind, rel_path))
+
+ return members
diff --git a/openviking/storage/ovpack/vectors.py b/openviking/storage/ovpack/vectors.py
new file mode 100644
index 000000000..2e2b425a5
--- /dev/null
+++ b/openviking/storage/ovpack/vectors.py
@@ -0,0 +1,327 @@
+# Copyright (c) 2026 Beijing Volcano Engine Technology Co., Ltd.
+# SPDX-License-Identifier: AGPL-3.0
+"""OVPack dense vector snapshot helpers."""
+
+from __future__ import annotations
+
+import hashlib
+import struct
+import zipfile
+from typing import Any
+
+from openviking.core.namespace import context_type_for_uri, owner_fields_for_uri
+from openviking.server.identity import RequestContext
+from openviking.storage.ovpack.format import (
+ OVPACK_DENSE_PATH,
+ dense_values_bytes,
+ internal_zip_path,
+ join_uri,
+ sha256_hex,
+)
+from openviking.storage.ovpack.manifest import manifest_dense_info
+from openviking.storage.ovpack.policy import NON_VECTOR_SCOPES
+from openviking.storage.ovpack.validation import dense_record_count, record_dense_ref
+from openviking.utils.time_utils import get_current_timestamp
+from openviking_cli.exceptions import InvalidArgumentError
+from openviking_cli.utils.logger import get_logger
+from openviking_cli.utils.uri import VikingURI
+
+logger = get_logger(__name__)
+
+
+def _index_meta_is_hybrid(index_meta: Any) -> bool:
+ if not isinstance(index_meta, dict):
+ return False
+ vector_index = index_meta.get("VectorIndex")
+ if not isinstance(vector_index, dict):
+ return False
+ index_type = str(vector_index.get("IndexType") or "").lower()
+ return "hybrid" in index_type
+
+
+def _vector_store_index_meta_is_hybrid(vector_store: Any) -> bool:
+ candidates = [vector_store, getattr(vector_store, "_manager", None)]
+ for candidate in candidates:
+ if candidate is None:
+ continue
+ index_name = getattr(candidate, "_index_name", None)
+ try:
+ backend = candidate._get_default_backend()
+ index_name = index_name or getattr(backend, "_index_name", None)
+ collection = backend._get_collection()
+ if index_name and hasattr(collection, "get_index_meta_data"):
+ if _index_meta_is_hybrid(collection.get_index_meta_data(index_name)):
+ return True
+ except Exception:
+ continue
+ return False
+
+
+def dense_snapshot_unsupported_reason(vector_store=None) -> str | None:
+ """Return why dense vector snapshots are unsupported in the current runtime."""
+ if _vector_store_index_meta_is_hybrid(vector_store):
+ return "current vector index type is hybrid"
+
+ return None
+
+
+def ensure_dense_snapshot_supported(vector_store=None) -> None:
+ reason = dense_snapshot_unsupported_reason(vector_store)
+ if reason:
+ raise InvalidArgumentError(
+ "ovpack vector snapshots only support pure dense vector indexes",
+ details={"reason": reason},
+ )
+
+
+def embedding_snapshot_metadata(dimensions: int | None) -> dict[str, Any]:
+ metadata: dict[str, Any] = {}
+ try:
+ from openviking_cli.utils.config import get_openviking_config
+
+ embedding_cfg = get_openviking_config().embedding
+ model_cfg = embedding_cfg.hybrid or embedding_cfg.dense
+ if model_cfg:
+ metadata = {
+ "provider": model_cfg.provider,
+ "model": model_cfg.model,
+ "input": model_cfg.input,
+ "query_param": model_cfg.query_param,
+ "document_param": model_cfg.document_param,
+ }
+ if dimensions is None:
+ dimensions = model_cfg.get_effective_dimension()
+ except Exception:
+ pass
+
+ if dimensions is not None:
+ metadata["dimensions"] = dimensions
+ return {key: value for key, value in metadata.items() if value is not None}
+
+
+def build_dense_snapshot_manifest(
+ index_records: list[dict[str, Any]],
+ dense_values: list[float],
+) -> tuple[bytes, dict[str, Any]] | None:
+ if not dense_values:
+ return None
+
+ dense_bytes = dense_values_bytes(dense_values)
+ dense_dimensions_set: set[int] = set()
+ dense_count = 0
+ for record in index_records:
+ dense = record_dense_ref(record)
+ if dense is None:
+ continue
+ dense_dimensions_set.add(dense["dimensions"])
+ dense_count += 1
+ if len(dense_dimensions_set) != 1:
+ raise InvalidArgumentError(
+ "Cannot export ovpack vectors with mixed dimensions",
+ details={"dimensions": sorted(dense_dimensions_set)},
+ )
+ dense_dimensions = next(iter(dense_dimensions_set))
+ return dense_bytes, {
+ "path": OVPACK_DENSE_PATH,
+ "count": dense_count,
+ "dtype": "float32",
+ "byte_order": "little",
+ "dimensions": dense_dimensions,
+ "sha256": sha256_hex(dense_bytes),
+ "embedding": embedding_snapshot_metadata(dense_dimensions),
+ }
+
+
+def read_dense_vectors(
+ zf: zipfile.ZipFile,
+ manifest: dict[str, Any],
+ base_name: str,
+ index_records: list[dict[str, Any]],
+) -> dict[str, list[float]]:
+ dense_info = manifest_dense_info(manifest)
+ if dense_info is None:
+ return {}
+ data = zf.read(internal_zip_path(base_name, OVPACK_DENSE_PATH))
+ vectors: dict[str, list[float]] = {}
+ for record in index_records:
+ record_id = record.get("record_id")
+ dense = record_dense_ref(record)
+ if not isinstance(record_id, str) or dense is None:
+ continue
+ offset = dense["offset"] * 4
+ dimensions = dense["dimensions"]
+ values = struct.unpack_from(f"<{dimensions}f", data, offset)
+ vectors[record_id] = [float(value) for value in values]
+ return vectors
+
+
+def current_embedding_metadata() -> dict[str, Any]:
+ try:
+ from openviking_cli.utils.config import get_openviking_config
+
+ embedding_cfg = get_openviking_config().embedding
+ model_cfg = embedding_cfg.hybrid or embedding_cfg.dense
+ if not model_cfg:
+ return {}
+ return {
+ "provider": model_cfg.provider,
+ "model": model_cfg.model,
+ "input": model_cfg.input,
+ "query_param": model_cfg.query_param,
+ "document_param": model_cfg.document_param,
+ "dimensions": model_cfg.get_effective_dimension(),
+ }
+ except Exception:
+ return {}
+
+
+def _embedding_snapshot_compatible(manifest: dict[str, Any]) -> tuple[bool, str]:
+ dense_info = manifest_dense_info(manifest)
+ if dense_info is None:
+ return False, "missing dense vector snapshot"
+
+ package_embedding = dense_info.get("embedding")
+ if not isinstance(package_embedding, dict):
+ return False, "missing embedding metadata"
+
+ current_embedding = current_embedding_metadata()
+ if not current_embedding:
+ return False, "current embedding metadata is unavailable"
+
+ fields = ("provider", "model", "input", "query_param", "document_param", "dimensions")
+ for field in fields:
+ package_value = package_embedding.get(field)
+ current_value = current_embedding.get(field)
+ if package_value != current_value:
+ return False, f"embedding {field} mismatch"
+ return True, ""
+
+
+def choose_vector_restore_action(
+ manifest: dict[str, Any],
+ index_records: list[dict[str, Any]],
+ dense_vectors: dict[str, list[float]],
+ *,
+ vector_store,
+ vector_mode: str,
+) -> str:
+ if vector_mode == "recompute":
+ return "recompute"
+
+ unsupported_reason = dense_snapshot_unsupported_reason(vector_store)
+ if unsupported_reason:
+ if vector_mode == "require":
+ raise InvalidArgumentError(
+ "ovpack dense vector snapshot cannot be restored into a sparse or hybrid index",
+ details={"reason": unsupported_reason},
+ )
+ logger.info(
+ "[ovpack] Recomputing vectors because dense snapshot restore is unsupported: "
+ f"{unsupported_reason}"
+ )
+ return "recompute"
+
+ dense_count = dense_record_count(index_records)
+ if dense_count == 0 or not dense_vectors:
+ if vector_mode == "require":
+ raise InvalidArgumentError(
+ "ovpack package does not contain a dense vector snapshot",
+ details={"vector_mode": vector_mode},
+ )
+ return "recompute"
+
+ if dense_count != len(dense_vectors):
+ if vector_mode == "require":
+ raise InvalidArgumentError("ovpack dense vector snapshot is incomplete")
+ return "recompute"
+
+ if not vector_store or not hasattr(vector_store, "upsert"):
+ if vector_mode == "require":
+ raise InvalidArgumentError("Vector restore requires a writable vector store")
+ return "recompute"
+
+ compatible, reason = _embedding_snapshot_compatible(manifest)
+ if not compatible:
+ if vector_mode == "require":
+ raise InvalidArgumentError(
+ "ovpack dense vector snapshot is incompatible with current embedding config",
+ details={"reason": reason},
+ )
+ logger.info(f"[ovpack] Recomputing vectors because snapshot is incompatible: {reason}")
+ return "recompute"
+
+ return "restore"
+
+
+def _vector_record_id(target_uri: str, level: int, ctx: RequestContext) -> str:
+ if level == 0:
+ seed_uri = (
+ target_uri if target_uri.endswith("/.abstract.md") else f"{target_uri}/.abstract.md"
+ )
+ elif level == 1:
+ seed_uri = (
+ target_uri if target_uri.endswith("/.overview.md") else f"{target_uri}/.overview.md"
+ )
+ else:
+ seed_uri = target_uri
+ return hashlib.md5(f"{ctx.account_id}:{seed_uri}".encode("utf-8")).hexdigest()
+
+
+async def _upsert_vector_snapshot_record(
+ vector_store,
+ target_uri: str,
+ record: dict[str, Any],
+ dense_vector: list[float],
+ ctx: RequestContext,
+) -> None:
+ level = int(record.get("level", 2))
+ scalars = dict(record.get("scalars") or {})
+ owner_fields = owner_fields_for_uri(target_uri, ctx=ctx)
+ timestamp = get_current_timestamp()
+ payload = {
+ **scalars,
+ "id": _vector_record_id(target_uri, level, ctx),
+ "uri": target_uri,
+ "context_type": context_type_for_uri(target_uri),
+ "level": level,
+ "created_at": timestamp,
+ "updated_at": timestamp,
+ "active_count": 0,
+ "account_id": ctx.account_id,
+ "owner_user_id": owner_fields.get("owner_user_id"),
+ "owner_agent_id": owner_fields.get("owner_agent_id"),
+ "vector": dense_vector,
+ }
+ if not payload.get("abstract"):
+ payload["abstract"] = str(record.get("text") or "")
+
+ try:
+ await vector_store.upsert(payload, ctx=ctx)
+ except TypeError:
+ await vector_store.upsert(payload)
+
+
+async def restore_vector_snapshot(
+ vector_store,
+ root_uri: str,
+ index_records: list[dict[str, Any]],
+ dense_vectors: dict[str, list[float]],
+ ctx: RequestContext,
+) -> None:
+ for record in index_records:
+ record_id = record.get("record_id")
+ if not isinstance(record_id, str) or record_id not in dense_vectors:
+ continue
+ rel_path = record.get("path")
+ if not isinstance(rel_path, str):
+ continue
+ target_uri = join_uri(root_uri, rel_path)
+ if VikingURI(target_uri).scope in NON_VECTOR_SCOPES:
+ continue
+ await _upsert_vector_snapshot_record(
+ vector_store,
+ target_uri,
+ record,
+ dense_vectors[record_id],
+ ctx,
+ )
diff --git a/openviking/sync_client.py b/openviking/sync_client.py
index 6940648fc..77c1c4872 100644
--- a/openviking/sync_client.py
+++ b/openviking/sync_client.py
@@ -293,19 +293,20 @@ class SyncOpenViking:
"""Delete relation"""
return run_async(self._async_client.unlink(from_uri, uri))
- def export_ovpack(self, uri: str, to: str) -> str:
+ def export_ovpack(self, uri: str, to: str, include_vectors: bool = False) -> str:
"""Export .ovpack file"""
- return run_async(self._async_client.export_ovpack(uri, to))
+ return run_async(self._async_client.export_ovpack(uri, to, include_vectors=include_vectors))
- def backup_ovpack(self, to: str) -> str:
+ def backup_ovpack(self, to: str, include_vectors: bool = False) -> str:
"""Back up public scopes as a restore-only .ovpack file."""
- return run_async(self._async_client.backup_ovpack(to))
+ return run_async(self._async_client.backup_ovpack(to, include_vectors=include_vectors))
def import_ovpack(
self,
file_path: str,
target: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Import .ovpack file (triggers vectorization by default)"""
return run_async(
@@ -313,6 +314,7 @@ class SyncOpenViking:
file_path,
target,
on_conflict=on_conflict,
+ vector_mode=vector_mode,
)
)
@@ -320,15 +322,21 @@ class SyncOpenViking:
self,
file_path: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Restore backup .ovpack file."""
return run_async(
self._async_client.restore_ovpack(
file_path,
on_conflict=on_conflict,
+ vector_mode=vector_mode,
)
)
+ def check_consistency(self, uri: str) -> Dict[str, Any]:
+ """Check filesystem/vector-index consistency for a URI subtree."""
+ return run_async(self._async_client.check_consistency(uri))
+
def close(self) -> None:
"""Close OpenViking and release resources."""
return run_async(self._async_client.close())
diff --git a/openviking_cli/client/base.py b/openviking_cli/client/base.py
index a25e10282..1b7d0dffd 100644
--- a/openviking_cli/client/base.py
+++ b/openviking_cli/client/base.py
@@ -298,12 +298,12 @@ class BaseClient(ABC):
# ============= Pack =============
@abstractmethod
- async def export_ovpack(self, uri: str, to: str) -> str:
+ async def export_ovpack(self, uri: str, to: str, include_vectors: bool = False) -> str:
"""Export as .ovpack file."""
...
@abstractmethod
- async def backup_ovpack(self, to: str) -> str:
+ async def backup_ovpack(self, to: str, include_vectors: bool = False) -> str:
"""Back up public scopes as a restore-only .ovpack file."""
...
@@ -313,6 +313,7 @@ class BaseClient(ABC):
file_path: str,
parent: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Import .ovpack file."""
...
@@ -322,12 +323,18 @@ class BaseClient(ABC):
self,
file_path: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Restore backup .ovpack file."""
...
# ============= Debug =============
+ @abstractmethod
+ async def check_consistency(self, uri: str) -> Dict[str, Any]:
+ """Check filesystem/vector-index consistency for a URI subtree."""
+ ...
+
@abstractmethod
async def health(self) -> bool:
"""Quick health check."""
diff --git a/openviking_cli/client/http.py b/openviking_cli/client/http.py
index 828922707..7634084d3 100644
--- a/openviking_cli/client/http.py
+++ b/openviking_cli/client/http.py
@@ -894,7 +894,12 @@ class AsyncHTTPClient(BaseClient):
# ============= Pack =============
- async def export_ovpack(self, uri: str, to: str) -> str:
+ async def export_ovpack(
+ self,
+ uri: str,
+ to: str,
+ include_vectors: bool = False,
+ ) -> str:
"""Export context as .ovpack file and save to local path.
Args:
@@ -922,7 +927,7 @@ class AsyncHTTPClient(BaseClient):
# Request export and stream response
response = await self._http.post(
"/api/v1/pack/export",
- json={"uri": uri},
+ json={"uri": uri, "include_vectors": include_vectors},
)
# Check for errors
@@ -935,7 +940,7 @@ class AsyncHTTPClient(BaseClient):
return str(to_path)
- async def backup_ovpack(self, to: str) -> str:
+ async def backup_ovpack(self, to: str, include_vectors: bool = False) -> str:
"""Back up public scopes as a restore-only .ovpack file."""
to_path = Path(to)
if to_path.is_dir():
@@ -945,7 +950,10 @@ class AsyncHTTPClient(BaseClient):
to_path.parent.mkdir(parents=True, exist_ok=True)
- response = await self._http.post("/api/v1/pack/backup", json={})
+ response = await self._http.post(
+ "/api/v1/pack/backup",
+ json={"include_vectors": include_vectors},
+ )
if not response.is_success:
self._handle_response(response)
@@ -959,6 +967,7 @@ class AsyncHTTPClient(BaseClient):
file_path: str,
parent: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Import .ovpack file."""
parent = VikingURI.normalize(parent)
@@ -967,6 +976,8 @@ class AsyncHTTPClient(BaseClient):
}
if on_conflict is not None:
request_data["on_conflict"] = on_conflict
+ if vector_mode is not None:
+ request_data["vector_mode"] = vector_mode
file_path_obj = Path(file_path)
if not file_path_obj.exists():
@@ -988,11 +999,14 @@ class AsyncHTTPClient(BaseClient):
self,
file_path: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Restore backup .ovpack file."""
request_data = {}
if on_conflict is not None:
request_data["on_conflict"] = on_conflict
+ if vector_mode is not None:
+ request_data["vector_mode"] = vector_mode
file_path_obj = Path(file_path)
if not file_path_obj.exists():
@@ -1012,6 +1026,14 @@ class AsyncHTTPClient(BaseClient):
# ============= Debug =============
+ async def check_consistency(self, uri: str) -> Dict[str, Any]:
+ """Check filesystem/vector-index consistency for a URI subtree."""
+ response = await self._http.post(
+ "/api/v1/system/consistency",
+ json={"uri": VikingURI.normalize(uri)},
+ )
+ return self._handle_response(response)
+
async def health(self) -> bool:
"""Check server health."""
try:
diff --git a/openviking_cli/client/sync_http.py b/openviking_cli/client/sync_http.py
index 293c900ba..9819f26db 100644
--- a/openviking_cli/client/sync_http.py
+++ b/openviking_cli/client/sync_http.py
@@ -407,7 +407,12 @@ class SyncHTTPClient:
# ============= Pack =============
- def export_ovpack(self, uri: str, to: str) -> str:
+ def export_ovpack(
+ self,
+ uri: str,
+ to: str,
+ include_vectors: bool = False,
+ ) -> str:
"""Export context as .ovpack file and save to local path.
Args:
@@ -417,17 +422,18 @@ class SyncHTTPClient:
Returns:
Local file path where the .ovpack was saved
"""
- return run_async(self._async_client.export_ovpack(uri, to))
+ return run_async(self._async_client.export_ovpack(uri, to, include_vectors=include_vectors))
- def backup_ovpack(self, to: str) -> str:
+ def backup_ovpack(self, to: str, include_vectors: bool = False) -> str:
"""Back up public scopes as a restore-only .ovpack file."""
- return run_async(self._async_client.backup_ovpack(to))
+ return run_async(self._async_client.backup_ovpack(to, include_vectors=include_vectors))
def import_ovpack(
self,
file_path: str,
target: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Import .ovpack file."""
return run_async(
@@ -435,6 +441,7 @@ class SyncHTTPClient:
file_path,
target,
on_conflict=on_conflict,
+ vector_mode=vector_mode,
)
)
@@ -442,15 +449,21 @@ class SyncHTTPClient:
self,
file_path: str,
on_conflict: Optional[str] = None,
+ vector_mode: Optional[str] = None,
) -> str:
"""Restore backup .ovpack file."""
return run_async(
self._async_client.restore_ovpack(
file_path,
on_conflict=on_conflict,
+ vector_mode=vector_mode,
)
)
+ def check_consistency(self, uri: str) -> Dict[str, Any]:
+ """Check filesystem/vector-index consistency for a URI subtree."""
+ return run_async(self._async_client.check_consistency(uri))
+
# ============= Admin =============
def admin_create_account(self, account_id: str, admin_user_id: str) -> Dict[str, Any]:
diff --git a/tests/client/test_import_export.py b/tests/client/test_import_export.py
index ae776529a..ff8263f5f 100644
--- a/tests/client/test_import_export.py
+++ b/tests/client/test_import_export.py
@@ -118,18 +118,32 @@ class TestImportOvpack:
@staticmethod
def _build_ovpack(zip_path: Path, entries: dict[str, str]) -> None:
+ index_records = b""
manifest = {
"kind": "openviking.ovpack",
"format_version": 2,
- "root": {"name": "pkg"},
+ "root": {
+ "name": "pkg",
+ "uri": "viking://resources/pkg",
+ "scope": "resources",
+ },
"entries": [{"path": "", "kind": "directory"}],
"content_sha256": hashlib.sha256(b"[]").hexdigest(),
- "vectors": {},
+ "index": {
+ "records": {
+ "path": "_ovpack/index_records.jsonl",
+ "count": 0,
+ "sha256": hashlib.sha256(index_records).hexdigest(),
+ }
+ },
}
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as zf:
zf.writestr("pkg/", "")
- zf.writestr("pkg/_._ovpack_manifest.json", json.dumps(manifest))
+ zf.writestr("pkg/files/", "")
+ zf.writestr("pkg/_ovpack/", "")
+ zf.writestr("pkg/_ovpack/index_records.jsonl", index_records)
+ zf.writestr("pkg/_ovpack/manifest.json", json.dumps(manifest))
for name, content in entries.items():
zf.writestr(name, content)
zip_path.write_bytes(buffer.getvalue())
diff --git a/tests/misc/test_ovpack_import_policy.py b/tests/misc/test_ovpack_import_policy.py
index 77f318406..e108d94ae 100644
--- a/tests/misc/test_ovpack_import_policy.py
+++ b/tests/misc/test_ovpack_import_policy.py
@@ -14,7 +14,13 @@ from pathlib import Path
import pytest
from openviking.server.identity import RequestContext, Role
-from openviking.storage.local_fs import backup_ovpack, export_ovpack, import_ovpack, restore_ovpack
+from openviking.storage.index_consistency import IndexConsistencyReport, IndexExpectation
+from openviking.storage.ovpack.operations import (
+ backup_ovpack,
+ export_ovpack,
+ import_ovpack,
+ restore_ovpack,
+)
from openviking_cli.exceptions import InvalidArgumentError, NotFoundError
from openviking_cli.session.user_id import UserIdentifier
@@ -97,6 +103,62 @@ class FakeExportVikingFS:
return self.binary_files[uri]
+class OverviewOnlyExportVikingFS(FakeExportVikingFS):
+ def __init__(self) -> None:
+ super().__init__()
+ self.text_files = {
+ "viking://resources/demo/.overview.md": "root overview",
+ }
+
+
+class ReservedPathExportVikingFS(FakeExportVikingFS):
+ def __init__(self) -> None:
+ super().__init__()
+ self.binary_files.update(
+ {
+ "viking://resources/demo/.ovpack/foo.txt": b"hello",
+ "viking://resources/demo/.notes.txt": b"dot",
+ "viking://resources/demo/_._notes.txt": b"escaped-looking",
+ }
+ )
+
+ async def tree(
+ self,
+ uri: str,
+ show_all_hidden: bool = False,
+ node_limit=None,
+ level_limit=None,
+ ctx=None,
+ ):
+ assert uri == "viking://resources/demo"
+ return [
+ {
+ "rel_path": ".ovpack",
+ "uri": "viking://resources/demo/.ovpack",
+ "isDir": True,
+ "size": 0,
+ },
+ {
+ "rel_path": ".ovpack/foo.txt",
+ "uri": "viking://resources/demo/.ovpack/foo.txt",
+ "isDir": False,
+ "size": 5,
+ },
+ {
+ "rel_path": ".notes.txt",
+ "uri": "viking://resources/demo/.notes.txt",
+ "isDir": False,
+ "size": 3,
+ },
+ {
+ "rel_path": "_._notes.txt",
+ "uri": "viking://resources/demo/_._notes.txt",
+ "isDir": False,
+ "size": 15,
+ },
+ ]
+
+
class FakeBackupVikingFS:
def __init__(self) -> None:
self.binary_files = {
@@ -151,6 +213,99 @@ class FakeBackupVikingFS:
return self.binary_files[uri]
+class FakeRestoreVectorVikingFS(FakeVikingFS):
+ async def tree(self, uri: str, node_limit=None, level_limit=None, ctx=None):
+ self.tree_calls.append(uri)
+ if uri == "viking://resources":
+ return [
+ {
+ "rel_path": "README.md",
+ "uri": "viking://resources/README.md",
+ "isDir": False,
+ "name": "README.md",
+ }
+ ]
+ return []
+
+
+class FakeVectorStore:
+ def __init__(self) -> None:
+ self.upserts: list[dict[str, object]] = []
+
+ async def filter(self, **kwargs):
+ uri = kwargs["filter"].value
+ if uri == "viking://resources/demo":
+ return [
+ {
+ "uri": uri,
+ "context_type": "resource",
+ "level": 0,
+ "abstract": "root abstract",
+ "vector": [0.4, 0.5, 0.6],
+ },
+ {
+ "uri": uri,
+ "context_type": "resource",
+ "level": 1,
+ "abstract": "root overview",
+ "vector": [0.7, 0.8, 0.9],
+ },
+ ]
+ if uri == "viking://resources/demo/notes.txt":
+ return [
+ {
+ "uri": uri,
+ "context_type": "resource",
+ "level": 2,
+ "abstract": "note summary",
+ "tags": ["snapshot"],
+ "vector": [0.1, 0.2, 0.3],
+ }
+ ]
+ return []
+
+ async def upsert(self, data, ctx=None):
+ self.upserts.append(dict(data))
+ return data.get("id", "")
+
+
+class IncompleteVectorStore(FakeVectorStore):
+ async def filter(self, **kwargs):
+ return []
+
+
+class OverviewOnlyVectorStore(FakeVectorStore):
+ async def filter(self, **kwargs):
+ uri = kwargs["filter"].value
+ if uri == "viking://resources/demo":
+ return [
+ {
+ "uri": uri,
+ "context_type": "resource",
+ "level": 1,
+ "abstract": "root overview",
+ "vector": [0.7, 0.8, 0.9],
+ }
+ ]
+ if uri == "viking://resources/demo/notes.txt":
+ return await super().filter(**kwargs)
+ return []
+
+
+class HybridIndexVectorStore(FakeVectorStore):
+ _index_name = "context_idx"
+
+ def _get_default_backend(self):
+ return self
+
+ def _get_collection(self):
+ return self
+
+ def get_index_meta_data(self, index_name):
+ assert index_name == self._index_name
+ return {"VectorIndex": {"IndexType": "flat_hybrid"}}
+
+
@pytest.fixture
def request_ctx() -> RequestContext:
return RequestContext(user=UserIdentifier("acct", "alice", "agent1"), role=Role.USER)
@@ -183,6 +338,31 @@ def _content_sha256(entries: list[dict[str, object]]) -> str:
return hashlib.sha256(payload).hexdigest()
+def _index_records_bytes(records: list[dict[str, object]]) -> bytes:
+ if not records:
+ return b""
+ lines = [
+ json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+ for record in records
+ ]
+ return ("\n".join(lines) + "\n").encode("utf-8")
+
+
+def _set_manifest_index(
+ manifest: dict[str, object], records: list[dict[str, object]] | None = None
+) -> bytes:
+ records = records or []
+ data = _index_records_bytes(records)
+ manifest["index"] = {
+ "records": {
+ "path": "_ovpack/index_records.jsonl",
+ "count": len(records),
+ "sha256": hashlib.sha256(data).hexdigest(),
+ }
+ }
+ return data
+
+
def _manifest_for_files(root_name: str, files: dict[str, str]) -> dict[str, object]:
entries: list[dict[str, object]] = [{"path": "", "kind": "directory"}]
content_entries: list[dict[str, object]] = []
@@ -203,7 +383,7 @@ def _manifest_for_files(root_name: str, files: dict[str, str]) -> dict[str, obje
}
)
- return {
+ manifest: dict[str, object] = {
"kind": "openviking.ovpack",
"format_version": 2,
"root": {
@@ -213,8 +393,9 @@ def _manifest_for_files(root_name: str, files: dict[str, str]) -> dict[str, obje
},
"entries": entries,
"content_sha256": _content_sha256(content_entries),
- "vectors": {},
}
+ _set_manifest_index(manifest)
+ return manifest
def _write_ovpack_with_manifest(
@@ -223,16 +404,81 @@ def _write_ovpack_with_manifest(
files: dict[str, str],
*,
manifest: dict[str, object] | None = None,
+ index_records: list[dict[str, object]] | None = None,
) -> None:
manifest = manifest or _manifest_for_files(root_name, files)
+ index_data = _set_manifest_index(manifest, index_records)
entries = {
f"{root_name}/": "",
- f"{root_name}/_._ovpack_manifest.json": json.dumps(manifest),
+ f"{root_name}/files/": "",
+ f"{root_name}/_ovpack/": "",
+ f"{root_name}/_ovpack/index_records.jsonl": index_data.decode("utf-8"),
+ f"{root_name}/_ovpack/manifest.json": json.dumps(manifest),
}
- entries.update({f"{root_name}/{rel_path}": content for rel_path, content in files.items()})
+ entries.update(
+ {f"{root_name}/files/{rel_path}": content for rel_path, content in files.items()}
+ )
_write_ovpack(path, entries)
+def _rewrite_ovpack_index(
+ path: Path,
+ root_name: str,
+ index_records: list[dict[str, object]],
+) -> None:
+ with zipfile.ZipFile(path, "r") as zf:
+ members = {info.filename: zf.read(info.filename) for info in zf.infolist()}
+
+ manifest_path = f"{root_name}/_ovpack/manifest.json"
+ index_path = f"{root_name}/_ovpack/index_records.jsonl"
+ manifest = json.loads(members[manifest_path].decode("utf-8"))
+ index_data = _set_manifest_index(manifest, index_records)
+ members[manifest_path] = json.dumps(manifest).encode("utf-8")
+ members[index_path] = index_data
+
+ with zipfile.ZipFile(path, "w") as zf:
+ for name, data in members.items():
+ zf.writestr(name, data)
+
+
+def _rewrite_ovpack_manifest(path: Path, root_name: str, mutate) -> None:
+ with zipfile.ZipFile(path, "r") as zf:
+ members = {info.filename: zf.read(info.filename) for info in zf.infolist()}
+
+ manifest_path = f"{root_name}/_ovpack/manifest.json"
+ manifest = json.loads(members[manifest_path].decode("utf-8"))
+ mutate(manifest)
+ members[manifest_path] = json.dumps(manifest).encode("utf-8")
+
+ with zipfile.ZipFile(path, "w") as zf:
+ for name, data in members.items():
+ zf.writestr(name, data)
+
+
+def test_index_consistency_report_limits_public_and_error_records():
+ missing = tuple(
+ IndexExpectation(
+ uri=f"viking://resources/demo/file_{index}.md",
+ rel_path=f"file_{index}.md",
+ level=2,
+ )
+ for index in range(25)
+ )
+ report = IndexConsistencyReport(expected=missing, missing_records=missing)
+
+ public = report.to_dict()
+ assert "expected" not in public
+ assert public["expected_count"] == 25
+ assert public["missing_record_count"] == 25
+ assert len(public["missing_records"]) == 20
+ assert public["missing_records_truncated"] is True
+
+ details = report.details()
+ assert details["missing_record_count"] == 25
+ assert details["missing_records"] == ["file_0.md#level=2"]
+ assert details["missing_records_truncated"] is True
+
+
@pytest.mark.asyncio
async def test_export_ovpack_writes_v2_manifest_with_semantic_sidecars(
temp_ovpack_path: Path, request_ctx: RequestContext
@@ -246,10 +492,14 @@ async def test_export_ovpack_writes_v2_manifest_with_semantic_sidecars(
with zipfile.ZipFile(temp_ovpack_path, "r") as zf:
names = set(zf.namelist())
- manifest = json.loads(zf.read("demo/_._ovpack_manifest.json").decode("utf-8"))
+ manifest = json.loads(zf.read("demo/_ovpack/manifest.json").decode("utf-8"))
+ index_records = [
+ json.loads(line)
+ for line in zf.read("demo/_ovpack/index_records.jsonl").decode("utf-8").splitlines()
+ ]
- assert "demo/notes.txt" in names
- assert "demo/_._overview.md" in names
+ assert "demo/files/notes.txt" in names
+ assert "demo/files/.overview.md" in names
assert manifest["format_version"] == 2
assert manifest["kind"] == "openviking.ovpack"
note_entry = next(entry for entry in manifest["entries"] if entry["path"] == "notes.txt")
@@ -265,7 +515,9 @@ async def test_export_ovpack_writes_v2_manifest_with_semantic_sidecars(
{"path": "notes.txt", "size": 5, "sha256": note_sha256},
]
)
- assert manifest["vectors"][""][0]["text"] == "root abstract"
+ assert manifest["index"]["records"]["count"] == len(index_records)
+ assert index_records[0]["path"] == ""
+ assert index_records[0]["text"] == "root abstract"
@pytest.mark.asyncio
@@ -278,10 +530,10 @@ async def test_backup_restore_contract(temp_ovpack_path: Path, request_ctx: Requ
with zipfile.ZipFile(temp_ovpack_path, "r") as zf:
names = set(zf.namelist())
- manifest = json.loads(zf.read("openviking-backup/_._ovpack_manifest.json").decode("utf-8"))
+ manifest = json.loads(zf.read("openviking-backup/_ovpack/manifest.json").decode("utf-8"))
- assert "openviking-backup/resources/README.md" in names
- assert "openviking-backup/session/sess_1/_._meta.json" in names
+ assert "openviking-backup/files/resources/README.md" in names
+ assert "openviking-backup/files/session/sess_1/.meta.json" in names
assert manifest["root"] == {
"name": "openviking-backup",
"uri": "viking://",
@@ -302,6 +554,220 @@ async def test_backup_restore_contract(temp_ovpack_path: Path, request_ctx: Requ
assert fake_fs.tree_calls == ["viking://resources", "viking://user", "viking://agent"]
+async def test_restore_ovpack_applies_backup_manifest_scalar_metadata(
+ temp_ovpack_path: Path, request_ctx: RequestContext, monkeypatch: pytest.MonkeyPatch
+):
+ await backup_ovpack(
+ FakeBackupVikingFS(),
+ str(temp_ovpack_path),
+ ctx=request_ctx,
+ )
+
+ _rewrite_ovpack_index(
+ temp_ovpack_path,
+ "openviking-backup",
+ [
+ {
+ "record_id": "r000001",
+ "path": "resources/README.md",
+ "kind": "file",
+ "level": 2,
+ "text": "portable summary",
+ "scalars": {
+ "abstract": "portable summary",
+ "description": "portable description",
+ "tags": ["portable"],
+ },
+ }
+ ],
+ )
+
+ vectorized_files: list[dict[str, object]] = []
+
+ async def capture_vectorize_file(**kwargs):
+ vectorized_files.append(kwargs)
+
+ monkeypatch.setattr(
+ "openviking.storage.ovpack.operations.vectorize_file", capture_vectorize_file
+ )
+
+ await restore_ovpack(FakeRestoreVectorVikingFS(), str(temp_ovpack_path), request_ctx)
+
+ assert len(vectorized_files) == 1
+ assert vectorized_files[0]["file_path"] == "viking://resources/README.md"
+ assert vectorized_files[0]["summary_dict"] == {
+ "name": "README.md",
+ "summary": "portable summary",
+ }
+ assert vectorized_files[0]["scalar_override"]["tags"] == ["portable"]
+
+
+async def test_export_include_vectors_rejects_missing_index_records(
+ temp_ovpack_path: Path, request_ctx: RequestContext
+):
+ with pytest.raises(
+ InvalidArgumentError,
+ match=r"incomplete OpenViking vector index snapshot",
+ ) as exc_info:
+ await export_ovpack(
+ FakeExportVikingFS(),
+ "viking://resources/demo",
+ str(temp_ovpack_path),
+ ctx=request_ctx,
+ vector_store=IncompleteVectorStore(),
+ include_vectors=True,
+ )
+
+ assert exc_info.value.details["missing_record_count"] == 3
+ assert exc_info.value.details["missing_records"] == [".#level=0"]
+ assert exc_info.value.details["missing_records_truncated"] is True
+ assert not temp_ovpack_path.read_bytes()
+
+
+async def test_export_include_vectors_allows_overview_without_abstract(
+ temp_ovpack_path: Path, request_ctx: RequestContext
+):
+ await export_ovpack(
+ OverviewOnlyExportVikingFS(),
+ "viking://resources/demo",
+ str(temp_ovpack_path),
+ ctx=request_ctx,
+ vector_store=OverviewOnlyVectorStore(),
+ include_vectors=True,
+ )
+
+ with zipfile.ZipFile(temp_ovpack_path, "r") as zf:
+ index_records = [
+ json.loads(line)
+ for line in zf.read("demo/_ovpack/index_records.jsonl").decode("utf-8").splitlines()
+ ]
+
+ root_levels = {record["level"] for record in index_records if record["path"] == ""}
+ assert root_levels == {1}
+
+
+async def test_ovpack_roundtrips_dot_and_escaped_looking_user_paths(
+ temp_ovpack_path: Path,
+ request_ctx: RequestContext,
+ monkeypatch: pytest.MonkeyPatch,
+):
+ async def noop_vectorization(*args, **kwargs):
+ return None
+
+ monkeypatch.setattr(
+ "openviking.storage.ovpack.operations._enqueue_direct_vectorization",
+ noop_vectorization,
+ )
+
+ await export_ovpack(
+ ReservedPathExportVikingFS(),
+ "viking://resources/demo",
+ str(temp_ovpack_path),
+ ctx=request_ctx,
+ )
+
+ with zipfile.ZipFile(temp_ovpack_path, "r") as zf:
+ names = set(zf.namelist())
+ manifest = json.loads(zf.read("demo/_ovpack/manifest.json").decode("utf-8"))
+
+ assert "demo/files/.ovpack/foo.txt" in names
+ assert "demo/files/.notes.txt" in names
+ assert "demo/files/_._notes.txt" in names
+ manifest_paths = {entry["path"] for entry in manifest["entries"]}
+ assert {".ovpack", ".ovpack/foo.txt", ".notes.txt", "_._notes.txt"} <= manifest_paths
+
+ fake_fs = FakeVikingFS()
+ await import_ovpack(fake_fs, str(temp_ovpack_path), "viking://resources/imported", request_ctx)
+
+ assert fake_fs.written_files == [
+ "viking://resources/imported/demo/.ovpack/foo.txt",
+ "viking://resources/imported/demo/.notes.txt",
+ "viking://resources/imported/demo/_._notes.txt",
+ ]
+
+
+async def test_export_include_vectors_rejects_hybrid_index_snapshot(
+ temp_ovpack_path: Path, request_ctx: RequestContext
+):
+ with pytest.raises(
+ InvalidArgumentError,
+ match=r"only support pure dense",
+ ) as exc_info:
+ await export_ovpack(
+ FakeExportVikingFS(),
+ "viking://resources/demo",
+ str(temp_ovpack_path),
+ ctx=request_ctx,
+ vector_store=HybridIndexVectorStore(),
+ include_vectors=True,
+ )
+
+ assert exc_info.value.details["reason"] == "current vector index type is hybrid"
+ assert not temp_ovpack_path.read_bytes()
+
+
+async def test_import_ovpack_restores_required_dense_vector_snapshot(
+ temp_ovpack_path: Path, request_ctx: RequestContext, monkeypatch: pytest.MonkeyPatch
+):
+ embedding_metadata = {
+ "provider": "test",
+ "model": "demo",
+ "input": "text",
+ "dimensions": 3,
+ }
+ monkeypatch.setattr(
+ "openviking.storage.ovpack.vectors.embedding_snapshot_metadata",
+ lambda dimensions: {**embedding_metadata, "dimensions": dimensions},
+ )
+ monkeypatch.setattr(
+ "openviking.storage.ovpack.vectors.current_embedding_metadata",
+ lambda: embedding_metadata,
+ )
+
+ await export_ovpack(
+ FakeExportVikingFS(),
+ "viking://resources/demo",
+ str(temp_ovpack_path),
+ ctx=request_ctx,
+ vector_store=FakeVectorStore(),
+ include_vectors=True,
+ )
+
+ with zipfile.ZipFile(temp_ovpack_path, "r") as zf:
+ assert "demo/_ovpack/dense.f32" in set(zf.namelist())
+
+ fake_fs = FakeVikingFS()
+ vector_store = FakeVectorStore()
+ result = await import_ovpack(
+ fake_fs,
+ str(temp_ovpack_path),
+ "viking://resources/imported",
+ request_ctx,
+ vector_mode="require",
+ vector_store=vector_store,
+ )
+
+ assert result == "viking://resources/imported/demo"
+ assert fake_fs.tree_calls == []
+ assert len(vector_store.upserts) == 3
+ note_record = next(
+ record
+ for record in vector_store.upserts
+ if record["uri"] == "viking://resources/imported/demo/notes.txt"
+ )
+ assert note_record["vector"] == pytest.approx([0.1, 0.2, 0.3])
+ assert note_record["tags"] == ["snapshot"]
+ assert note_record["created_at"]
+ assert note_record["updated_at"]
+ assert note_record["active_count"] == 0
+
+
@pytest.mark.asyncio
async def test_import_legacy_ovpack_without_manifest_is_rejected(
temp_ovpack_path: Path, request_ctx: RequestContext
@@ -309,8 +775,8 @@ async def test_import_legacy_ovpack_without_manifest_is_rejected(
_write_ovpack(
temp_ovpack_path,
{
- "demo/_._overview.md": "ATTACKER_OVERVIEW",
- "demo/notes.txt": "hello",
+ "demo/files/.overview.md": "ATTACKER_OVERVIEW",
+ "demo/files/notes.txt": "hello",
},
)
fake_fs = FakeVikingFS()
@@ -364,9 +830,12 @@ async def test_import_ovpack_rejects_manifest_unexpected_directory(
temp_ovpack_path,
{
"demo/": "",
- "demo/_._ovpack_manifest.json": json.dumps(manifest),
- "demo/notes.txt": "hello",
- "demo/empty/": "",
+ "demo/files/": "",
+ "demo/_ovpack/": "",
+ "demo/_ovpack/index_records.jsonl": "",
+ "demo/_ovpack/manifest.json": json.dumps(manifest),
+ "demo/files/notes.txt": "hello",
+ "demo/files/empty/": "",
},
)
fake_fs = FakeVikingFS()
@@ -460,3 +929,14 @@ async def test_import_top_level_scope_package_requires_root_target(
assert await import_ovpack(fake_fs, str(temp_ovpack_path), "viking://", request_ctx) == (
"viking://resources"
)
+
+ _write_ovpack_with_manifest(
+ temp_ovpack_path,
+ "renamed",
+ {"README.md": "hello"},
+ manifest=manifest,
+ )
+ with pytest.raises(InvalidArgumentError, match=r"root name does not match zip root"):
+ await import_ovpack(
+ FakeVikingFS(), str(temp_ovpack_path), "viking://resources", request_ctx
+ )
diff --git a/tests/server/ovpack_test_helpers.py b/tests/server/ovpack_test_helpers.py
index f9c4b15f4..83eb56ebb 100644
--- a/tests/server/ovpack_test_helpers.py
+++ b/tests/server/ovpack_test_helpers.py
@@ -19,6 +19,17 @@ def _content_sha256(entries: list[dict[str, object]]) -> str:
return hashlib.sha256(payload).hexdigest()
+def _index_records_bytes(records: list[dict[str, object]] | None = None) -> bytes:
+ records = records or []
+ if not records:
+ return b""
+ lines = [
+ json.dumps(record, ensure_ascii=False, sort_keys=True, separators=(",", ":"))
+ for record in records
+ ]
+ return ("\n".join(lines) + "\n").encode("utf-8")
+
+
def build_ovpack_bytes(
root_name: str = "pkg",
files: dict[str, bytes] | None = None,
@@ -44,19 +55,33 @@ def build_ovpack_bytes(
}
)
+ index_records = _index_records_bytes()
manifest = {
"kind": "openviking.ovpack",
"format_version": 2,
- "root": {"name": root_name},
+ "root": {
+ "name": root_name,
+ "uri": f"viking://resources/{root_name}",
+ "scope": "resources",
+ },
"entries": manifest_entries,
"content_sha256": _content_sha256(content_entries),
- "vectors": {},
+ "index": {
+ "records": {
+ "path": "_ovpack/index_records.jsonl",
+ "count": 0,
+ "sha256": hashlib.sha256(index_records).hexdigest(),
+ }
+ },
}
buffer = io.BytesIO()
with zipfile.ZipFile(buffer, "w") as zf:
zf.writestr(f"{root_name}/", "")
- zf.writestr(f"{root_name}/_._ovpack_manifest.json", json.dumps(manifest))
+ zf.writestr(f"{root_name}/files/", "")
+ zf.writestr(f"{root_name}/_ovpack/", "")
+ zf.writestr(f"{root_name}/_ovpack/index_records.jsonl", index_records)
+ zf.writestr(f"{root_name}/_ovpack/manifest.json", json.dumps(manifest))
for rel_path, content in files.items():
- zf.writestr(f"{root_name}/{rel_path}", content)
+ zf.writestr(f"{root_name}/files/{rel_path}", content)
return buffer.getvalue()