Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
fix: #1238 and #1242 and #1232 (#1243)
MaojiaSheng4c175f25367a
diff --git a/bot/vikingbot/config/loader.py b/bot/vikingbot/config/loader.pyindex 9aae2f826..bc49b0a4d 100644--- a/bot/vikingbot/config/loader.py+++ b/bot/vikingbot/config/loader.py@@ -4,11 +4,14 @@ import jsonimport osfrom pathlib import Pathfrom typing import Any+from loguru import logger+from vikingbot.config.schema import ConfigCONFIG_PATH = None+def get_config_path() -> Path:"""Get the path to ov.conf config file.@@ -24,9 +27,7 @@ def _resolve_ov_conf_path() -> Path:# Check environment variable firstenv_path = os.environ.get("OPENVIKING_CONFIG_FILE")if env_path:- path = Path(env_path).expanduser()- if path.exists():- return path+ return Path(env_path).expanduser()# Default pathreturn Path.home() / ".openviking" / "ov.conf"@@ -222,4 +223,4 @@ def camel_to_snake(name: str) -> str:def snake_to_camel(name: str) -> str:"""Convert snake_case to camelCase."""components = name.split("_")- return components[0] + "".join(x.title() for x in components[1:])\ No newline at end of file+ return components[0] + "".join(x.title() for x in components[1:])diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rsindex 5525475f3..aed1ffef4 100644--- a/crates/ov_cli/src/client.rs+++ b/crates/ov_cli/src/client.rs@@ -732,12 +732,73 @@ impl HttpClient {// ============ Pack Methods ============- pub async fn export_ovpack(&self, uri: &str, to: &str) -> Result<serde_json::Value> {+ pub async fn export_ovpack(&self, uri: &str, to: &str) -> Result<String> {let body = serde_json::json!({"uri": uri,- "to": to,});- self.post("/api/v1/pack/export", &body).await++ let url = format!("{}/api/v1/pack/export", self.base_url);+ let response = self+ .http+ .post(&url)+ .headers(self.build_headers())+ .json(&body)+ .send()+ .await+ .map_err(|e| Error::Network(format!("HTTP request failed: {}", e)))?;++ let status = response.status();+ if !status.is_success() {+ // Try to parse error message as JSON+ let json_result: Result<serde_json::Value> = response+ .json()+ .await+ .map_err(|e| Error::Network(format!("Failed to parse error response: {}", e)));++ let error_msg = match json_result {+ Ok(json) => json+ .get("error")+ .and_then(|e| e.get("message"))+ .and_then(|m| m.as_str())+ .map(|s| s.to_string())+ .or_else(|| {+ json.get("detail")+ .and_then(|d| d.as_str())+ .map(|s| s.to_string())+ })+ .unwrap_or_else(|| format!("HTTP error {}", status)),+ Err(_) => format!("HTTP error {}", status),+ };++ return Err(Error::Api(error_msg));+ }++ // Download the file content+ let bytes = response+ .bytes()+ .await+ .map_err(|e| Error::Network(format!("Failed to read response bytes: {}", e)))?;++ // Determine target path+ let to_path = Path::new(to);+ let final_path = if to_path.is_dir() {+ let base_name = uri.trim_end_matches('/').split('/').last().unwrap_or("export");+ to_path.join(format!("{}.ovpack", base_name))+ } else if !to.ends_with(".ovpack") {+ Path::new(&format!("{}.ovpack", to)).to_path_buf()+ } else {+ to_path.to_path_buf()+ };++ // Ensure parent directory exists+ if let Some(parent) = final_path.parent() {+ std::fs::create_dir_all(parent)?;+ }++ // Write file+ std::fs::write(&final_path, bytes)?;++ Ok(final_path.to_string_lossy().to_string())}pub async fn import_ovpack(diff --git a/crates/ov_cli/src/commands/pack.rs b/crates/ov_cli/src/commands/pack.rsindex 82b5f2f2a..54341659a 100644--- a/crates/ov_cli/src/commands/pack.rs+++ b/crates/ov_cli/src/commands/pack.rs@@ -9,7 +9,14 @@ pub async fn export(format: OutputFormat,compact: bool,) -> Result<()> {- let result = client.export_ovpack(uri, to).await?;+ let file_path = client.export_ovpack(uri, to).await?;++ // Output success message with the file path+ let result = serde_json::json!({+ "file": file_path,+ "message": format!("Successfully exported to {}", file_path)+ });+output_success(&result, format, compact);Ok(())}diff --git a/openviking/models/embedder/openai_embedders.py b/openviking/models/embedder/openai_embedders.pyindex 0e67069de..45ef82715 100644--- a/openviking/models/embedder/openai_embedders.py+++ b/openviking/models/embedder/openai_embedders.py@@ -85,7 +85,9 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):non-symmetric mode with query_param/document_param.api_key: API key, if None will read from env vars (OPENVIKING_EMBEDDING_API_KEY or OPENAI_API_KEY)api_base: API base URL, optional. Required for third-party OpenAI-compatible APIs.- dimension: Dimension (if model supports), optional+ dimension: Target dimension for output vectors. If specified and the model returns vectors+ with a different dimension, the output will be truncated to this dimension.+ If None, uses the model's default dimension without truncation.query_param: Parameter for query-side embeddings. Supports simple values (e.g., 'query')or key=value format (e.g., 'input_type=query,task=search'). Defaults to None.Setting this (or document_param) activates non-symmetric mode.@@ -139,6 +141,7 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):# Auto-detect dimensionself._dimension = dimension+ self._actual_model_dimension = Noneif self._dimension is None:self._dimension = self._detect_dimension()@@ -146,11 +149,26 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):"""Detect dimension by making an actual API call"""try:result = self.embed("test")- return len(result.dense_vector) if result.dense_vector else 1536+ detected_dim = len(result.dense_vector) if result.dense_vector else 1536+ self._actual_model_dimension = detected_dim+ return detected_dimexcept Exception:# Use default value, text-embedding-3-small defaults to 1536return 1536+ def _truncate_vector(self, vector: List[float]) -> List[float]:+ """Truncate vector to target dimension if needed.++ Args:+ vector: Input vector from API++ Returns:+ Truncated vector if dimension is set and smaller than input, otherwise original vector+ """+ if self.dimension is not None and len(vector) > self.dimension:+ return vector[: self.dimension]+ return vector+def _update_telemetry_token_usage(self, response) -> None:usage = getattr(response, "usage", None)if not usage:@@ -251,8 +269,8 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):def _call() -> EmbedResult:kwargs: Dict[str, Any] = {"input": text, "model": self.model_name}- if self.dimension:- kwargs["dimensions"] = self.dimension+ # Don't pass dimensions parameter to API - some OpenAI-compatible models don't support it+ # Instead, we'll truncate the result vector if neededextra_body = self._build_extra_body(is_query=is_query)if extra_body:@@ -262,6 +280,9 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):self._update_telemetry_token_usage(response)vector = response.data[0].embedding+ # Truncate vector if needed+ vector = self._truncate_vector(vector)+return EmbedResult(dense_vector=vector)try:@@ -293,8 +314,8 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):def _call() -> List[EmbedResult]:kwargs: Dict[str, Any] = {"input": texts, "model": self.model_name}- if self.dimension:- kwargs["dimensions"] = self.dimension+ # Don't pass dimensions parameter to API - some OpenAI-compatible models don't support it+ # Instead, we'll truncate the result vectors if neededextra_body = self._build_extra_body(is_query=is_query)if extra_body:@@ -303,7 +324,11 @@ class OpenAIDenseEmbedder(DenseEmbedderBase):response = self.client.embeddings.create(**kwargs)self._update_telemetry_token_usage(response)- return [EmbedResult(dense_vector=item.embedding) for item in response.data]+ # Truncate vectors if needed+ return [+ EmbedResult(dense_vector=self._truncate_vector(item.embedding))+ for item in response.data+ ]try:return self._run_with_retry(diff --git a/openviking/server/routers/pack.py b/openviking/server/routers/pack.pyindex 8b70fa145..7984183be 100644--- a/openviking/server/routers/pack.py+++ b/openviking/server/routers/pack.py@@ -2,8 +2,13 @@# SPDX-License-Identifier: AGPL-3.0"""Pack endpoints for OpenViking HTTP Server."""+import os+import tempfile+from fastapi import APIRouter, Depends+from fastapi.responses import FileResponsefrom pydantic import BaseModel, ConfigDict+from starlette.background import BackgroundTaskfrom openviking.server.auth import get_request_contextfrom openviking.server.dependencies import get_service@@ -19,7 +24,6 @@ class ExportRequest(BaseModel):"""Request model for export."""uri: str- to: strclass ImportRequest(BaseModel):@@ -45,10 +49,40 @@ async def export_ovpack(request: ExportRequest,_ctx: RequestContext = Depends(get_request_context),):- """Export context as .ovpack file."""+ """Export context as .ovpack file and stream it to client."""service = get_service()- result = await service.pack.export_ovpack(request.uri, request.to, ctx=_ctx)- return Response(status="ok", result={"file": result})++ # Create temp file for export+ temp_dir = tempfile.gettempdir()+ temp_file = os.path.join(temp_dir, f"export_{os.urandom(16).hex()}.ovpack")++ try:+ # Export to temp file+ await service.pack.export_ovpack(request.uri, temp_file, ctx=_ctx)++ # Determine filename from URI+ base_name = request.uri.strip().rstrip("/").split("/")[-1]+ if not base_name:+ base_name = "export"+ filename = f"{base_name}.ovpack"++ # Create background task for cleanup+ def cleanup():+ if os.path.exists(temp_file):+ os.unlink(temp_file)++ # Stream file back to client with cleanup+ return FileResponse(+ path=temp_file,+ media_type="application/zip",+ filename=filename,+ background=BackgroundTask(cleanup),+ )+ except Exception:+ # Clean up temp file on error+ if os.path.exists(temp_file):+ os.unlink(temp_file)+ raise@router.post("/import")diff --git a/openviking/storage/local_fs.py b/openviking/storage/local_fs.pyindex a42810e1f..483eac004 100644--- a/openviking/storage/local_fs.py+++ b/openviking/storage/local_fs.py@@ -249,7 +249,14 @@ async def export_ovpack(viking_fs, uri: str, to: str, ctx: RequestContext) -> stReturns:Exported file path++ Raises:+ ValueError: If export size exceeds limits (65536 files or 2GB total size)"""+ # Safety limits+ MAX_FILES = 65536+ MAX_TOTAL_SIZE = 2 * 1024 * 1024 * 1024 # 2GB+base_name = uri.strip().rstrip("/").split("/")[-1]if not base_name:base_name = "export"@@ -263,6 +270,30 @@ async def export_ovpack(viking_fs, uri: str, to: str, ctx: RequestContext) -> stentries = await viking_fs.tree(uri, show_all_hidden=True, ctx=ctx)+ # Check file count limit+ file_count = sum(1 for entry in entries if not entry.get("isDir"))+ if file_count > MAX_FILES:+ raise ValueError(+ f"Export aborted: too many files ({file_count} files, limit is {MAX_FILES}). "+ f"Please export a smaller directory."+ )++ # Calculate total size and check limit+ total_size = 0+ for entry in entries:+ if not entry.get("isDir"):+ # Get file size from entry if available+ size = entry.get("size", 0)+ total_size += size++ if total_size > MAX_TOTAL_SIZE:+ size_mb = total_size / (1024 * 1024)+ limit_mb = MAX_TOTAL_SIZE / (1024 * 1024)+ raise ValueError(+ f"Export aborted: total size too large ({size_mb:.1f}MB, limit is {limit_mb:.0f}MB). "+ f"Please export a smaller directory."+ )+with zipfile.ZipFile(to, "w", zipfile.ZIP_DEFLATED, allowZip64=True) as zf:# Write root directory entryzf.writestr(base_name + "/", "")diff --git a/openviking_cli/client/http.py b/openviking_cli/client/http.pyindex 587b2b627..edc0852fd 100644--- a/openviking_cli/client/http.py+++ b/openviking_cli/client/http.py@@ -829,14 +829,45 @@ class AsyncHTTPClient(BaseClient):# ============= Pack =============async def export_ovpack(self, uri: str, to: str) -> str:- """Export context as .ovpack file."""+ """Export context as .ovpack file and save to local path.++ Args:+ uri: Viking URI to export+ to: Local file path where to save the .ovpack file++ Returns:+ Local file path where the .ovpack was saved+ """uri = VikingURI.normalize(uri)++ # Determine target path+ to_path = Path(to)+ if to_path.is_dir():+ base_name = uri.strip().rstrip("/").split("/")[-1]+ if not base_name:+ base_name = "export"+ to_path = to_path / f"{base_name}.ovpack"+ elif not str(to_path).endswith(".ovpack"):+ to_path = Path(str(to_path) + ".ovpack")++ # Ensure parent directory exists+ to_path.parent.mkdir(parents=True, exist_ok=True)++ # Request export and stream responseresponse = await self._http.post("/api/v1/pack/export",- json={"uri": uri, "to": to},+ json={"uri": uri},)- result = self._handle_response(response)- return result.get("file", "")++ # Check for errors+ if not response.is_success:+ self._handle_response(response)++ # Save streamed content to local file+ with open(to_path, "wb") as f:+ f.write(response.content)++ return str(to_path)async def import_ovpack(self,diff --git a/openviking_cli/client/sync_http.py b/openviking_cli/client/sync_http.pyindex 9b0f20dae..d5cf7a301 100644--- a/openviking_cli/client/sync_http.py+++ b/openviking_cli/client/sync_http.py@@ -256,7 +256,9 @@ class SyncHTTPClient:exclude_uri: Optional[str] = None,) -> Dict:"""Content search with pattern."""- return run_async(self._async_client.grep(uri, pattern, case_insensitive, node_limit, exclude_uri))+ return run_async(+ self._async_client.grep(uri, pattern, case_insensitive, node_limit, exclude_uri)+ )def glob(self, pattern: str, uri: str = "viking://") -> Dict:"""File pattern matching."""@@ -374,7 +376,15 @@ class SyncHTTPClient:# ============= Pack =============def export_ovpack(self, uri: str, to: str) -> str:- """Export context as .ovpack file."""+ """Export context as .ovpack file and save to local path.++ Args:+ uri: Viking URI to export+ to: Local file path where to save the .ovpack file++ Returns:+ Local file path where the .ovpack was saved+ """return run_async(self._async_client.export_ovpack(uri, to))def import_ovpack(diff --git a/tests/api_test/api/client.py b/tests/api_test/api/client.pyindex c5d4b1972..1994e84f5 100644--- a/tests/api_test/api/client.py+++ b/tests/api_test/api/client.py@@ -377,9 +377,29 @@ class OpenVikingAPIClient:return self.session.get(url)def export_ovpack(self, uri: str, to: str) -> requests.Response:+ """Export ovpack and save to local file path.++ Args:+ uri: Viking URI to export+ to: Local file path where to save the .ovpack file++ Returns:+ Response object with the downloaded file saved to 'to' path+ """endpoint = "/api/v1/pack/export"url = self._build_url(self.server_url, endpoint)- return self.session.post(url, json={"uri": uri, "to": to})++ # Request export (server streams the file)+ response = self._request_with_retry("POST", url, json={"uri": uri})++ # Save streamed content to local file+ if response.status_code == 200:+ to_path = Path(to)+ to_path.parent.mkdir(parents=True, exist_ok=True)+ with open(to_path, "wb") as f:+ f.write(response.content)++ return responsedef import_ovpack(self, file_path: str, parent: str, force: bool = False, vectorize: bool = Truediff --git a/tests/api_test/resources/test_pack.py b/tests/api_test/resources/test_pack.pyindex 74a6de7dd..4d4a3a9e7 100644--- a/tests/api_test/resources/test_pack.py+++ b/tests/api_test/resources/test_pack.py@@ -1,4 +1,4 @@-import json+import osimport uuid@@ -27,16 +27,17 @@ class TestPack:response = api_client.export_ovpack(uri=test_uri, to=test_export_path)print(f"\nExport ovpack API status code: {response.status_code}")- data = response.json()- print("\n" + "=" * 80)- print("Export OVPack API Response:")- print("=" * 80)- print(json.dumps(data, indent=2, ensure_ascii=False))- print("=" * 80 + "\n")+ # The response is now a file stream, not JSON+ assert response.status_code == 200, f"Expected status 200, got {response.status_code}"- assert data.get("status") == "ok", f"Expected status 'ok', got {data.get('status')}"- assert data.get("error") is None, f"Expected error to be null, got {data.get('error')}"- assert "result" in data, "'result' field should exist"+ # Verify the file was created+ assert os.path.exists(test_export_path), (+ f"Export file not created at {test_export_path}"+ )+ assert os.path.getsize(test_export_path) > 0, "Export file is empty"++ print(f"\nSuccessfully exported to {test_export_path}")+ print(f"File size: {os.path.getsize(test_export_path)} bytes")except Exception as e:print(f"Error: {e}")