Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.
Python agent-contextagent-memoryagent-pluginsagentic-ragcontext-databasedsh-pluginknowledge-basememorymemory-managementself-evolvingvector-database
feat: break change, remove is_leaf scalar and use level instead (#271)
MaojiaShenga412c7d16464
diff --git a/README.md b/README.mdindex bed8fb71a..9ef267d0b 100644--- a/README.md+++ b/README.md@@ -205,10 +205,17 @@ vllm serve meta-llama/Llama-3.1-8B-Instruct --port 8000#### Configuration Template-Create a configuration file `~/.openviking/ov.conf`:+Create a configuration file `~/.openviking/ov.conf`, remove the comments before copy:```json{+ "storage": {+ "workspace": "/home/your-name/openviking_workspace"+ },+ "log": {+ "level": "INFO",+ "output": "stdout" // Log output: "stdout" or "file"+ },"embedding": {"dense": {"api_base" : "<api-endpoint>", // API endpoint address@@ -238,6 +245,13 @@ Create a configuration file `~/.openviking/ov.conf`:```json{+ "storage": {+ "workspace": "/home/your-name/openviking_workspace"+ },+ "log": {+ "level": "INFO",+ "output": "stdout" // Log output: "stdout" or "file"+ },"embedding": {"dense": {"api_base" : "https://ark.cn-beijing.volces.com/api/v3",@@ -263,6 +277,13 @@ Create a configuration file `~/.openviking/ov.conf`:```json{+ "storage": {+ "workspace": "/home/your-name/openviking_workspace"+ },+ "log": {+ "level": "INFO",+ "output": "stdout" // Log output: "stdout" or "file"+ },"embedding": {"dense": {"api_base" : "https://api.openai.com/v1",diff --git a/README_CN.md b/README_CN.mdindex b2f8875ff..3c55e6bb4 100644--- a/README_CN.md+++ b/README_CN.md@@ -87,6 +87,13 @@ OpenViking 支持多种模型服务:```json{+ "storage": {+ "workspace": "/home/your-name/openviking_workspace"+ },+ "log": {+ "level": "INFO",+ "output": "stdout" // 可配置为 "stdout" or "file"+ },"embedding": {"dense": {"api_base" : "<api-endpoint>", // API 服务端点地址@@ -114,6 +121,13 @@ OpenViking 支持多种模型服务:```json{+ "storage": {+ "workspace": "/home/your-name/openviking_workspace"+ },+ "log": {+ "level": "INFO",+ "output": "stdout" // 可配置为 "stdout" or "file"+ },"embedding": {"dense": {"api_base" : "https://ark.cn-beijing.volces.com/api/v3",@@ -139,6 +153,13 @@ OpenViking 支持多种模型服务:```json{+ "storage": {+ "workspace": "/home/your-name/openviking_workspace"+ },+ "log": {+ "level": "INFO",+ "output": "stdout" // 可配置为 "stdout" or "file"+ },"embedding": {"dense": {"api_base" : "https://api.openai.com/v1",diff --git a/crates/ov_cli/Cargo.toml b/crates/ov_cli/Cargo.tomlindex 5207d08c9..48d3df510 100644--- a/crates/ov_cli/Cargo.toml+++ b/crates/ov_cli/Cargo.toml@@ -24,3 +24,7 @@ thiserror = "1.0"unicode-width = "0.1"ratatui = "0.30"crossterm = "0.28"+zip = "2.2"+tempfile = "3.12"+url = "2.5"+walkdir = "2.5"diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rsindex 37a492b35..abfef95b1 100644--- a/crates/ov_cli/src/client.rs+++ b/crates/ov_cli/src/client.rs@@ -1,6 +1,12 @@use reqwest::{Client as ReqwestClient, StatusCode};use serde::de::DeserializeOwned;use serde_json::Value;+use std::fs::File;+use std::path::Path;+use tempfile::NamedTempFile;+use url::Url;+use zip::write::FileOptions;+use zip::CompressionMethod;use crate::error::{Error, Result};@@ -27,6 +33,82 @@ impl HttpClient {}}+ /// Check if the server is localhost or 127.0.0.1+ fn is_local_server(&self) -> bool {+ if let Ok(url) = Url::parse(&self.base_url) {+ if let Some(host) = url.host_str() {+ return host == "localhost" || host == "127.0.0.1";+ }+ }+ false+ }++ /// Zip a directory to a temporary file+ fn zip_directory(&self, dir_path: &Path) -> Result<NamedTempFile> {+ if !dir_path.is_dir() {+ return Err(Error::Network(format!(+ "Path {} is not a directory",+ dir_path.display()+ )));+ }++ let temp_file = NamedTempFile::new()?;+ let file = File::create(temp_file.path())?;+ let mut zip = zip::ZipWriter::new(file);+ let options: FileOptions<'_, ()> = FileOptions::default().compression_method(CompressionMethod::Deflated);++ let walkdir = walkdir::WalkDir::new(dir_path);+ for entry in walkdir.into_iter().filter_map(|e| e.ok()) {+ let path = entry.path();+ if path.is_file() {+ let name = path.strip_prefix(dir_path).unwrap_or(path);+ zip.start_file(name.to_string_lossy(), options)?;+ let mut file = File::open(path)?;+ std::io::copy(&mut file, &mut zip)?;+ }+ }++ zip.finish()?;+ Ok(temp_file)+ }++ /// Upload a temporary file and return the temp_path+ async fn upload_temp_file(&self, file_path: &Path) -> Result<String> {+ let url = format!("{}/api/v1/resources/temp_upload", self.base_url);+ let file_name = file_path+ .file_name()+ .and_then(|n| n.to_str())+ .unwrap_or("temp_upload.zip");++ // Read file content+ let file_content = tokio::fs::read(file_path).await?;++ // Create multipart form+ let part = reqwest::multipart::Part::bytes(file_content)+ .file_name(file_name.to_string());++ let part = part.mime_str("application/octet-stream").map_err(|e| {+ Error::Network(format!("Failed to set mime type: {}", e))+ })?;++ let form = reqwest::multipart::Form::new().part("file", part);++ let response = self+ .http+ .post(&url)+ .multipart(form)+ .send()+ .await+ .map_err(|e| Error::Network(format!("HTTP request failed: {}", e)))?;++ let result: Value = self.handle_response(response).await?;+ result+ .get("temp_path")+ .and_then(|v| v.as_str())+ .map(|s| s.to_string())+ .ok_or_else(|| Error::Parse("Missing temp_path in response".to_string()))+ }+fn build_headers(&self) -> reqwest::header::HeaderMap {let mut headers = reqwest::header::HeaderMap::new();headers.insert(@@ -327,15 +409,37 @@ impl HttpClient {wait: bool,timeout: Option<f64>,) -> Result<serde_json::Value> {- let body = serde_json::json!({- "path": path,- "target": target,- "reason": reason,- "instruction": instruction,- "wait": wait,- "timeout": timeout,- });- self.post("/api/v1/resources", &body).await+ let path_obj = Path::new(path);++ // Check if it's a local directory and not a local server+ if path_obj.exists() && path_obj.is_dir() && !self.is_local_server() {+ // Zip the directory+ let zip_file = self.zip_directory(path_obj)?;+ let temp_path = self.upload_temp_file(zip_file.path()).await?;++ let body = serde_json::json!({+ "temp_path": temp_path,+ "target": target,+ "reason": reason,+ "instruction": instruction,+ "wait": wait,+ "timeout": timeout,+ });++ self.post("/api/v1/resources", &body).await+ } else {+ // Regular case - use path directly+ let body = serde_json::json!({+ "path": path,+ "target": target,+ "reason": reason,+ "instruction": instruction,+ "wait": wait,+ "timeout": timeout,+ });++ self.post("/api/v1/resources", &body).await+ }}pub async fn add_skill(diff --git a/crates/ov_cli/src/error.rs b/crates/ov_cli/src/error.rsindex bdbdd9d48..9a2df34ac 100644--- a/crates/ov_cli/src/error.rs+++ b/crates/ov_cli/src/error.rs@@ -25,6 +25,9 @@ pub enum Error {#[error("Serialization error: {0}")]Serialization(#[from] serde_json::Error),++ #[error("Zip error: {0}")]+ Zip(#[from] zip::result::ZipError),}pub type Result<T> = std::result::Result<T, Error>;@@ -75,6 +78,7 @@ impl From<Error> for CliError {Error::Output(msg) => CliError::new(format!("Output error: {}", msg)),Error::Io(e) => CliError::new(format!("IO error: {}", e)),Error::Serialization(e) => CliError::new(format!("Serialization error: {}", e)),+ Error::Zip(e) => CliError::new(format!("Zip error: {}", e)),}}}@@ -84,7 +88,7 @@ impl From<reqwest::Error> for CliError {if err.is_connect() || err.is_timeout() {CliError::network(format!("Failed to connect to OpenViking server. \- Check the url in ovcli.conf and ensure the server is running. ({ })",+ Check the url in ovcli.conf and ensure the server is running. ({})",err))} else {diff --git a/docs/en/guides/03-deployment.md b/docs/en/guides/03-deployment.mdindex cb5fa86af..9f57a764d 100644--- a/docs/en/guides/03-deployment.md+++ b/docs/en/guides/03-deployment.md@@ -94,6 +94,59 @@ Server connects to remote AGFS and VectorDB services. Configure remote URLs in `python -m openviking serve```+## Deploying with Systemd (Recommended)++For Linux systems, you can use Systemd to manage OpenViking as a service, enabling automatic restart and startup on boot. Firstly, you should tried to install and configure openviking on your own.++### Create Systemd Service File++Create `/etc/systemd/system/openviking.service` file:++```ini+[Unit]+Description=OpenViking HTTP Server+After=network.target++[Service]+Type=simple+# Replace with the user running OpenViking+User=your-username+# Replace with the user group+Group=your-group+# Replace with your working directory+WorkingDirectory=/home/your-username/openviking_workspace+# Choose one of the following start methods+ExecStart=/path/to/your/python/bin/openviking-server+Restart=always+RestartSec=5+# Path to config file+Environment="OPENVIKING_CONFIG_FILE=/home/your-username/.openviking/ov.conf"++[Install]+WantedBy=multi-user.target+```++### Manage the Service++After creating the service file, use the following commands to manage the OpenViking service:++```bash+# Reload systemd configuration+sudo systemctl daemon-reload++# Start the service+sudo systemctl start openviking.service++# Enable service on boot+sudo systemctl enable openviking.service++# Check service status+sudo systemctl status openviking.service++# View service logs+sudo journalctl -u openviking.service -f+```+## Connecting Clients### Python SDKdiff --git a/docs/zh/guides/03-deployment.md b/docs/zh/guides/03-deployment.mdindex 1e15e4efb..ccc7c9cea 100644--- a/docs/zh/guides/03-deployment.md+++ b/docs/zh/guides/03-deployment.md@@ -94,6 +94,59 @@ python -m openviking servepython -m openviking serve```+## 使用 Systemd 部署服务(推荐)++对于 Linux 系统,可以使用 Systemd 服务来管理 OpenViking,实现自动重启、开机自启等功能。首先,你应该已经成功安装并配置了 OpenViking 服务器,确保它可以正常运行,再进行服务化部署。++### 创建 Systemd 服务文件++创建 `/etc/systemd/system/openviking.service` 文件:++```ini+[Unit]+Description=OpenViking HTTP Server+After=network.target++[Service]+Type=simple+# 替换为运行 OpenViking 的用户+User=your-username+# 替换为用户组+Group=your-group+# 替换为工作目录+WorkingDirectory=/home/your-username/openviking_workspace+# 以下两种启动方式二选一+ExecStart=/path/to/your/python/bin/openviking-server+Restart=always+RestartSec=5+# 配置文件路径+Environment="OPENVIKING_CONFIG_FILE=/home/your-username/.openviking/ov.conf"++[Install]+WantedBy=multi-user.target+```++### 管理服务++创建好服务文件后,使用以下命令管理 OpenViking 服务:++```bash+# 重载 systemd 配置+sudo systemctl daemon-reload++# 启动服务+sudo systemctl start openviking.service++# 设置开机自启+sudo systemctl enable openviking.service++# 查看服务状态+sudo systemctl status openviking.service++# 查看服务日志+sudo journalctl -u openviking.service -f+```+## 连接客户端### Python SDKdiff --git a/openviking/core/context.py b/openviking/core/context.pyindex ba816085b..f22d8491d 100644--- a/openviking/core/context.py+++ b/openviking/core/context.py@@ -29,6 +29,14 @@ class ContextType(str, Enum):RESOURCE = "resource"+class ContextLevel(int, Enum):+ """Context level (L0/L1/L2) for vector indexing"""++ ABSTRACT = 0 # L0: abstract+ OVERVIEW = 1 # L1: overview+ DETAIL = 2 # L2: detail/content++class Vectorize:text: str = ""# image: str = ""diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.pyindex d99a9a8dd..26b4372a5 100644--- a/openviking/retrieve/hierarchical_retriever.py+++ b/openviking/retrieve/hierarchical_retriever.py@@ -191,7 +191,7 @@ class HierarchicalRetriever:global_filter = {"op": "and",- "conds": [filter, {"op": "must", "field": "is_leaf", "conds": [False]}],+ "conds": [filter, {"op": "must", "field": "level", "conds": [0, 1]}],}results = await self.storage.search(collection=collection,@@ -351,7 +351,7 @@ class HierarchicalRetriever:)if uri not in visited:- if r.get("is_leaf"):+ if r.get("level") == 2:visited.add(uri)else:heapq.heappush(dir_queue, (-final_score, uri))@@ -400,7 +400,7 @@ class HierarchicalRetriever:MatchedContext(uri=c.get("uri", ""),context_type=context_type,- is_leaf=c.get("is_leaf", False),+ level=c.get("level", 2),abstract=c.get("abstract", ""),category=c.get("category", ""),score=c.get("_final_score", c.get("_score", 0.0)),diff --git a/openviking/server/bootstrap.py b/openviking/server/bootstrap.pyindex e818464d9..4dd6c3341 100644--- a/openviking/server/bootstrap.py+++ b/openviking/server/bootstrap.py@@ -58,6 +58,7 @@ def main():# Create and run appapp = create_app(config)+ print(f"OpenViking HTTP Server is running on {config.host}:{config.port}")uvicorn.run(app, host=config.host, port=config.port, log_config=None)diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.pyindex c703b33bd..61b76d364 100644--- a/openviking/storage/collection_schemas.py+++ b/openviking/storage/collection_schemas.py@@ -45,7 +45,15 @@ class CollectionSchemas:"Fields": [{"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},{"FieldName": "uri", "FieldType": "path"},+ # type 字段:当前版本未使用,保留用于未来扩展+ # 预留用于表示资源的具体类型,如 "file", "directory", "image", "video", "repository" 等{"FieldName": "type", "FieldType": "string"},+ # context_type 字段:区分上下文的大类+ # 枚举值:"resource"(资源,默认), "memory"(记忆), "skill"(技能)+ # 推导规则:+ # - URI 以 viking://agent/skills 开头 → "skill"+ # - URI 包含 "memories" → "memory"+ # - 其他情况 → "resource"{"FieldName": "context_type", "FieldType": "string"},{"FieldName": "vector", "FieldType": "vector", "Dim": vector_dim},{"FieldName": "sparse_vector", "FieldType": "sparse_vector"},@@ -53,7 +61,16 @@ class CollectionSchemas:{"FieldName": "updated_at", "FieldType": "date_time"},{"FieldName": "active_count", "FieldType": "int64"},{"FieldName": "parent_uri", "FieldType": "path"},- {"FieldName": "is_leaf", "FieldType": "bool"},+ # level 字段:区分 L0/L1/L2 层级+ # 枚举值:+ # - 0 = L0(abstract,摘要)+ # - 1 = L1(overview,概览)+ # - 2 = L2(detail/content,详情/内容,默认)+ # URI 命名规则:+ # - level=0: {目录}/.abstract.md+ # - level=1: {目录}/.overview.md+ # - level=2: {文件路径}+ {"FieldName": "level", "FieldType": "int64"},{"FieldName": "name", "FieldType": "string"},{"FieldName": "description", "FieldType": "string"},{"FieldName": "tags", "FieldType": "string"},@@ -67,7 +84,7 @@ class CollectionSchemas:"updated_at","active_count","parent_uri",- "is_leaf",+ "level","name","tags",],diff --git a/openviking/storage/queuefs/embedding_msg_converter.py b/openviking/storage/queuefs/embedding_msg_converter.pyindex f0980bf35..d105d5f60 100644--- a/openviking/storage/queuefs/embedding_msg_converter.py+++ b/openviking/storage/queuefs/embedding_msg_converter.py@@ -7,7 +7,7 @@ This module provides a unified interface for converting Context objectsto EmbeddingMsg objects for asynchronous vector processing."""-from openviking.core.context import Context+from openviking.core.context import Context, ContextLevelfrom openviking.storage.queuefs.embedding_msg import EmbeddingMsgfrom openviking_cli.utils import get_logger@@ -26,9 +26,20 @@ class EmbeddingMsgConverter:if not vectorization_text:return None+ context_dict = context.to_dict()++ # 根据 URI 判断 level 字段(用于向量索引)+ uri = context_dict.get("uri", "")+ if uri.endswith("/.abstract.md"):+ context_dict["level"] = ContextLevel.ABSTRACT+ elif uri.endswith("/.overview.md"):+ context_dict["level"] = ContextLevel.OVERVIEW+ else:+ context_dict["level"] = ContextLevel.DETAIL+embedding_msg = EmbeddingMsg(message=vectorization_text,- context_data=context.to_dict(),+ context_data=context_dict,)# Set any additional fields from kwargsdiff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.pyindex cfad652d2..d312c57ad 100644--- a/openviking/storage/queuefs/semantic_processor.py+++ b/openviking/storage/queuefs/semantic_processor.py@@ -441,21 +441,36 @@ class SemanticProcessor(DequeueHandlerBase):from openviking.storage.queuefs import get_queue_managerfrom openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter- parent_uri = VikingURI(uri).parent.uri- context = Context(- uri=uri,- parent_uri=parent_uri,+ queue_manager = get_queue_manager()+ embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)++ # Vectorize L0: .abstract.md (abstract), level=0+ abstract_uri = f"{uri}/.abstract.md"+ context_abstract = Context(+ uri=abstract_uri,+ parent_uri=uri,is_leaf=False,abstract=abstract,context_type=context_type,)- context.set_vectorize(Vectorize(text=overview))-- embedding_msg = EmbeddingMsgConverter.from_context(context)- queue_manager = get_queue_manager()- embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)- await embedding_queue.enqueue(embedding_msg) # type: ignore- logger.debug(f"Enqueued directory for vectorization: {uri}")+ context_abstract.set_vectorize(Vectorize(text=abstract))+ embedding_msg_abstract = EmbeddingMsgConverter.from_context(context_abstract)+ await embedding_queue.enqueue(embedding_msg_abstract) # type: ignore+ logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {abstract_uri}")++ # Vectorize L1: .overview.md (overview), level=1+ overview_uri = f"{uri}/.overview.md"+ context_overview = Context(+ uri=overview_uri,+ parent_uri=uri,+ is_leaf=False,+ abstract=abstract,+ context_type=context_type,+ )+ context_overview.set_vectorize(Vectorize(text=overview))+ embedding_msg_overview = EmbeddingMsgConverter.from_context(context_overview)+ await embedding_queue.enqueue(embedding_msg_overview) # type: ignore+ logger.debug(f"Enqueued directory L1 (overview) for vectorization: {overview_uri}")async def _vectorize_files(self,diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.pyindex bcfbdf1c6..d75a55237 100644--- a/openviking/storage/viking_vector_index_backend.py+++ b/openviking/storage/viking_vector_index_backend.py@@ -574,7 +574,7 @@ class VikingVectorIndexBackend(VikingDBInterface):total_deleted = 0# If any record indicates this URI is a directory node, remove descendants first.- if any(not r.get("is_leaf", False) for r in target_records):+ if any(r.get("level") in [0, 1] for r in target_records):descendant_count = await self._remove_descendants(collection, uri)total_deleted += descendant_count@@ -602,10 +602,10 @@ class VikingVectorIndexBackend(VikingDBInterface):for child in children:child_uri = child.get("uri")- is_leaf = child.get("is_leaf", False)+ level = child.get("level", 2)# Recursively delete if child is also an intermediate directory- if not is_leaf and child_uri:+ if level in [0, 1] and child_uri:descendant_count = await self._remove_descendants(collection, child_uri)total_deleted += descendant_countdiff --git a/openviking_cli/retrieve/types.py b/openviking_cli/retrieve/types.pyindex 9d22a8d9a..db4347f0a 100644--- a/openviking_cli/retrieve/types.py+++ b/openviking_cli/retrieve/types.py@@ -283,7 +283,7 @@ class MatchedContext:uri: strcontext_type: ContextType- is_leaf: bool = False+ level: int = 2abstract: str = ""overview: Optional[str] = Nonecategory: str = ""@@ -367,7 +367,7 @@ class FindResult:return {"context_type": ctx.context_type.value,"uri": ctx.uri,- "is_leaf": ctx.is_leaf,+ "level": ctx.level,"score": ctx.score,"category": ctx.category,"match_reason": ctx.match_reason,@@ -393,7 +393,7 @@ class FindResult:return MatchedContext(uri=d.get("uri", ""),context_type=ContextType(d.get("context_type", "resource")),- is_leaf=d.get("is_leaf", False),+ level=d.get("level", 2),abstract=d.get("abstract", ""),overview=d.get("overview"),category=d.get("category", ""),