Skip to content

Self-evolving Context Database for AI Agents. Unify Agent Memory, Knowledge RAG and Skills.

feat: break change, remove is_leaf scalar and use level instead (#271)

MaojiaShenga412c7d16464
diff --git a/README.md b/README.md
index bed8fb71a..9ef267d0b 100644
--- a/README.md
+++ b/README.md
@@ -205,10 +205,17 @@ vllm serve meta-llama/Llama-3.1-8B-Instruct --port 8000
#### Configuration Template
-Create a configuration file `~/.openviking/ov.conf`:
+Create a configuration file `~/.openviking/ov.conf`, remove the comments before copy:
```json
{
+ "storage": {
+ "workspace": "/home/your-name/openviking_workspace"
+ },
+ "log": {
+ "level": "INFO",
+ "output": "stdout" // Log output: "stdout" or "file"
+ },
"embedding": {
"dense": {
"api_base" : "<api-endpoint>", // API endpoint address
@@ -238,6 +245,13 @@ Create a configuration file `~/.openviking/ov.conf`:
```json
{
+ "storage": {
+ "workspace": "/home/your-name/openviking_workspace"
+ },
+ "log": {
+ "level": "INFO",
+ "output": "stdout" // Log output: "stdout" or "file"
+ },
"embedding": {
"dense": {
"api_base" : "https://ark.cn-beijing.volces.com/api/v3",
@@ -263,6 +277,13 @@ Create a configuration file `~/.openviking/ov.conf`:
```json
{
+ "storage": {
+ "workspace": "/home/your-name/openviking_workspace"
+ },
+ "log": {
+ "level": "INFO",
+ "output": "stdout" // Log output: "stdout" or "file"
+ },
"embedding": {
"dense": {
"api_base" : "https://api.openai.com/v1",
diff --git a/README_CN.md b/README_CN.md
index b2f8875ff..3c55e6bb4 100644
--- a/README_CN.md
+++ b/README_CN.md
@@ -87,6 +87,13 @@ OpenViking 支持多种模型服务:
```json
{
+ "storage": {
+ "workspace": "/home/your-name/openviking_workspace"
+ },
+ "log": {
+ "level": "INFO",
+ "output": "stdout" // 可配置为 "stdout" or "file"
+ },
"embedding": {
"dense": {
"api_base" : "<api-endpoint>", // API 服务端点地址
@@ -114,6 +121,13 @@ OpenViking 支持多种模型服务:
```json
{
+ "storage": {
+ "workspace": "/home/your-name/openviking_workspace"
+ },
+ "log": {
+ "level": "INFO",
+ "output": "stdout" // 可配置为 "stdout" or "file"
+ },
"embedding": {
"dense": {
"api_base" : "https://ark.cn-beijing.volces.com/api/v3",
@@ -139,6 +153,13 @@ OpenViking 支持多种模型服务:
```json
{
+ "storage": {
+ "workspace": "/home/your-name/openviking_workspace"
+ },
+ "log": {
+ "level": "INFO",
+ "output": "stdout" // 可配置为 "stdout" or "file"
+ },
"embedding": {
"dense": {
"api_base" : "https://api.openai.com/v1",
diff --git a/crates/ov_cli/Cargo.toml b/crates/ov_cli/Cargo.toml
index 5207d08c9..48d3df510 100644
--- a/crates/ov_cli/Cargo.toml
+++ b/crates/ov_cli/Cargo.toml
@@ -24,3 +24,7 @@ thiserror = "1.0"
unicode-width = "0.1"
ratatui = "0.30"
crossterm = "0.28"
+zip = "2.2"
+tempfile = "3.12"
+url = "2.5"
+walkdir = "2.5"
diff --git a/crates/ov_cli/src/client.rs b/crates/ov_cli/src/client.rs
index 37a492b35..abfef95b1 100644
--- a/crates/ov_cli/src/client.rs
+++ b/crates/ov_cli/src/client.rs
@@ -1,6 +1,12 @@
use reqwest::{Client as ReqwestClient, StatusCode};
use serde::de::DeserializeOwned;
use serde_json::Value;
+use std::fs::File;
+use std::path::Path;
+use tempfile::NamedTempFile;
+use url::Url;
+use zip::write::FileOptions;
+use zip::CompressionMethod;
use crate::error::{Error, Result};
@@ -27,6 +33,82 @@ impl HttpClient {
}
}
+ /// Check if the server is localhost or 127.0.0.1
+ fn is_local_server(&self) -> bool {
+ if let Ok(url) = Url::parse(&self.base_url) {
+ if let Some(host) = url.host_str() {
+ return host == "localhost" || host == "127.0.0.1";
+ }
+ }
+ false
+ }
+
+ /// Zip a directory to a temporary file
+ fn zip_directory(&self, dir_path: &Path) -> Result<NamedTempFile> {
+ if !dir_path.is_dir() {
+ return Err(Error::Network(format!(
+ "Path {} is not a directory",
+ dir_path.display()
+ )));
+ }
+
+ let temp_file = NamedTempFile::new()?;
+ let file = File::create(temp_file.path())?;
+ let mut zip = zip::ZipWriter::new(file);
+ let options: FileOptions<'_, ()> = FileOptions::default().compression_method(CompressionMethod::Deflated);
+
+ let walkdir = walkdir::WalkDir::new(dir_path);
+ for entry in walkdir.into_iter().filter_map(|e| e.ok()) {
+ let path = entry.path();
+ if path.is_file() {
+ let name = path.strip_prefix(dir_path).unwrap_or(path);
+ zip.start_file(name.to_string_lossy(), options)?;
+ let mut file = File::open(path)?;
+ std::io::copy(&mut file, &mut zip)?;
+ }
+ }
+
+ zip.finish()?;
+ Ok(temp_file)
+ }
+
+ /// Upload a temporary file and return the temp_path
+ async fn upload_temp_file(&self, file_path: &Path) -> Result<String> {
+ let url = format!("{}/api/v1/resources/temp_upload", self.base_url);
+ let file_name = file_path
+ .file_name()
+ .and_then(|n| n.to_str())
+ .unwrap_or("temp_upload.zip");
+
+ // Read file content
+ let file_content = tokio::fs::read(file_path).await?;
+
+ // Create multipart form
+ let part = reqwest::multipart::Part::bytes(file_content)
+ .file_name(file_name.to_string());
+
+ let part = part.mime_str("application/octet-stream").map_err(|e| {
+ Error::Network(format!("Failed to set mime type: {}", e))
+ })?;
+
+ let form = reqwest::multipart::Form::new().part("file", part);
+
+ let response = self
+ .http
+ .post(&url)
+ .multipart(form)
+ .send()
+ .await
+ .map_err(|e| Error::Network(format!("HTTP request failed: {}", e)))?;
+
+ let result: Value = self.handle_response(response).await?;
+ result
+ .get("temp_path")
+ .and_then(|v| v.as_str())
+ .map(|s| s.to_string())
+ .ok_or_else(|| Error::Parse("Missing temp_path in response".to_string()))
+ }
+
fn build_headers(&self) -> reqwest::header::HeaderMap {
let mut headers = reqwest::header::HeaderMap::new();
headers.insert(
@@ -327,15 +409,37 @@ impl HttpClient {
wait: bool,
timeout: Option<f64>,
) -> Result<serde_json::Value> {
- let body = serde_json::json!({
- "path": path,
- "target": target,
- "reason": reason,
- "instruction": instruction,
- "wait": wait,
- "timeout": timeout,
- });
- self.post("/api/v1/resources", &body).await
+ let path_obj = Path::new(path);
+
+ // Check if it's a local directory and not a local server
+ if path_obj.exists() && path_obj.is_dir() && !self.is_local_server() {
+ // Zip the directory
+ let zip_file = self.zip_directory(path_obj)?;
+ let temp_path = self.upload_temp_file(zip_file.path()).await?;
+
+ let body = serde_json::json!({
+ "temp_path": temp_path,
+ "target": target,
+ "reason": reason,
+ "instruction": instruction,
+ "wait": wait,
+ "timeout": timeout,
+ });
+
+ self.post("/api/v1/resources", &body).await
+ } else {
+ // Regular case - use path directly
+ let body = serde_json::json!({
+ "path": path,
+ "target": target,
+ "reason": reason,
+ "instruction": instruction,
+ "wait": wait,
+ "timeout": timeout,
+ });
+
+ self.post("/api/v1/resources", &body).await
+ }
}
pub async fn add_skill(
diff --git a/crates/ov_cli/src/error.rs b/crates/ov_cli/src/error.rs
index bdbdd9d48..9a2df34ac 100644
--- a/crates/ov_cli/src/error.rs
+++ b/crates/ov_cli/src/error.rs
@@ -25,6 +25,9 @@ pub enum Error {
#[error("Serialization error: {0}")]
Serialization(#[from] serde_json::Error),
+
+ #[error("Zip error: {0}")]
+ Zip(#[from] zip::result::ZipError),
}
pub type Result<T> = std::result::Result<T, Error>;
@@ -75,6 +78,7 @@ impl From<Error> for CliError {
Error::Output(msg) => CliError::new(format!("Output error: {}", msg)),
Error::Io(e) => CliError::new(format!("IO error: {}", e)),
Error::Serialization(e) => CliError::new(format!("Serialization error: {}", e)),
+ Error::Zip(e) => CliError::new(format!("Zip error: {}", e)),
}
}
}
@@ -84,7 +88,7 @@ impl From<reqwest::Error> for CliError {
if err.is_connect() || err.is_timeout() {
CliError::network(format!(
"Failed to connect to OpenViking server. \
- Check the url in ovcli.conf and ensure the server is running. ({ })",
+ Check the url in ovcli.conf and ensure the server is running. ({})",
err
))
} else {
diff --git a/docs/en/guides/03-deployment.md b/docs/en/guides/03-deployment.md
index cb5fa86af..9f57a764d 100644
--- a/docs/en/guides/03-deployment.md
+++ b/docs/en/guides/03-deployment.md
@@ -94,6 +94,59 @@ Server connects to remote AGFS and VectorDB services. Configure remote URLs in `
python -m openviking serve
```
+## Deploying with Systemd (Recommended)
+
+For Linux systems, you can use Systemd to manage OpenViking as a service, enabling automatic restart and startup on boot. Firstly, you should tried to install and configure openviking on your own.
+
+### Create Systemd Service File
+
+Create `/etc/systemd/system/openviking.service` file:
+
+```ini
+[Unit]
+Description=OpenViking HTTP Server
+After=network.target
+
+[Service]
+Type=simple
+# Replace with the user running OpenViking
+User=your-username
+# Replace with the user group
+Group=your-group
+# Replace with your working directory
+WorkingDirectory=/home/your-username/openviking_workspace
+# Choose one of the following start methods
+ExecStart=/path/to/your/python/bin/openviking-server
+Restart=always
+RestartSec=5
+# Path to config file
+Environment="OPENVIKING_CONFIG_FILE=/home/your-username/.openviking/ov.conf"
+
+[Install]
+WantedBy=multi-user.target
+```
+
+### Manage the Service
+
+After creating the service file, use the following commands to manage the OpenViking service:
+
+```bash
+# Reload systemd configuration
+sudo systemctl daemon-reload
+
+# Start the service
+sudo systemctl start openviking.service
+
+# Enable service on boot
+sudo systemctl enable openviking.service
+
+# Check service status
+sudo systemctl status openviking.service
+
+# View service logs
+sudo journalctl -u openviking.service -f
+```
+
## Connecting Clients
### Python SDK
diff --git a/docs/zh/guides/03-deployment.md b/docs/zh/guides/03-deployment.md
index 1e15e4efb..ccc7c9cea 100644
--- a/docs/zh/guides/03-deployment.md
+++ b/docs/zh/guides/03-deployment.md
@@ -94,6 +94,59 @@ python -m openviking serve
python -m openviking serve
```
+## 使用 Systemd 部署服务(推荐)
+
+对于 Linux 系统,可以使用 Systemd 服务来管理 OpenViking,实现自动重启、开机自启等功能。首先,你应该已经成功安装并配置了 OpenViking 服务器,确保它可以正常运行,再进行服务化部署。
+
+### 创建 Systemd 服务文件
+
+创建 `/etc/systemd/system/openviking.service` 文件:
+
+```ini
+[Unit]
+Description=OpenViking HTTP Server
+After=network.target
+
+[Service]
+Type=simple
+# 替换为运行 OpenViking 的用户
+User=your-username
+# 替换为用户组
+Group=your-group
+# 替换为工作目录
+WorkingDirectory=/home/your-username/openviking_workspace
+# 以下两种启动方式二选一
+ExecStart=/path/to/your/python/bin/openviking-server
+Restart=always
+RestartSec=5
+# 配置文件路径
+Environment="OPENVIKING_CONFIG_FILE=/home/your-username/.openviking/ov.conf"
+
+[Install]
+WantedBy=multi-user.target
+```
+
+### 管理服务
+
+创建好服务文件后,使用以下命令管理 OpenViking 服务:
+
+```bash
+# 重载 systemd 配置
+sudo systemctl daemon-reload
+
+# 启动服务
+sudo systemctl start openviking.service
+
+# 设置开机自启
+sudo systemctl enable openviking.service
+
+# 查看服务状态
+sudo systemctl status openviking.service
+
+# 查看服务日志
+sudo journalctl -u openviking.service -f
+```
+
## 连接客户端
### Python SDK
diff --git a/openviking/core/context.py b/openviking/core/context.py
index ba816085b..f22d8491d 100644
--- a/openviking/core/context.py
+++ b/openviking/core/context.py
@@ -29,6 +29,14 @@ class ContextType(str, Enum):
RESOURCE = "resource"
+class ContextLevel(int, Enum):
+ """Context level (L0/L1/L2) for vector indexing"""
+
+ ABSTRACT = 0 # L0: abstract
+ OVERVIEW = 1 # L1: overview
+ DETAIL = 2 # L2: detail/content
+
+
class Vectorize:
text: str = ""
# image: str = ""
diff --git a/openviking/retrieve/hierarchical_retriever.py b/openviking/retrieve/hierarchical_retriever.py
index d99a9a8dd..26b4372a5 100644
--- a/openviking/retrieve/hierarchical_retriever.py
+++ b/openviking/retrieve/hierarchical_retriever.py
@@ -191,7 +191,7 @@ class HierarchicalRetriever:
global_filter = {
"op": "and",
- "conds": [filter, {"op": "must", "field": "is_leaf", "conds": [False]}],
+ "conds": [filter, {"op": "must", "field": "level", "conds": [0, 1]}],
}
results = await self.storage.search(
collection=collection,
@@ -351,7 +351,7 @@ class HierarchicalRetriever:
)
if uri not in visited:
- if r.get("is_leaf"):
+ if r.get("level") == 2:
visited.add(uri)
else:
heapq.heappush(dir_queue, (-final_score, uri))
@@ -400,7 +400,7 @@ class HierarchicalRetriever:
MatchedContext(
uri=c.get("uri", ""),
context_type=context_type,
- is_leaf=c.get("is_leaf", False),
+ level=c.get("level", 2),
abstract=c.get("abstract", ""),
category=c.get("category", ""),
score=c.get("_final_score", c.get("_score", 0.0)),
diff --git a/openviking/server/bootstrap.py b/openviking/server/bootstrap.py
index e818464d9..4dd6c3341 100644
--- a/openviking/server/bootstrap.py
+++ b/openviking/server/bootstrap.py
@@ -58,6 +58,7 @@ def main():
# Create and run app
app = create_app(config)
+ print(f"OpenViking HTTP Server is running on {config.host}:{config.port}")
uvicorn.run(app, host=config.host, port=config.port, log_config=None)
diff --git a/openviking/storage/collection_schemas.py b/openviking/storage/collection_schemas.py
index c703b33bd..61b76d364 100644
--- a/openviking/storage/collection_schemas.py
+++ b/openviking/storage/collection_schemas.py
@@ -45,7 +45,15 @@ class CollectionSchemas:
"Fields": [
{"FieldName": "id", "FieldType": "string", "IsPrimaryKey": True},
{"FieldName": "uri", "FieldType": "path"},
+ # type 字段:当前版本未使用,保留用于未来扩展
+ # 预留用于表示资源的具体类型,如 "file", "directory", "image", "video", "repository" 等
{"FieldName": "type", "FieldType": "string"},
+ # context_type 字段:区分上下文的大类
+ # 枚举值:"resource"(资源,默认), "memory"(记忆), "skill"(技能)
+ # 推导规则:
+ # - URI 以 viking://agent/skills 开头 → "skill"
+ # - URI 包含 "memories" → "memory"
+ # - 其他情况 → "resource"
{"FieldName": "context_type", "FieldType": "string"},
{"FieldName": "vector", "FieldType": "vector", "Dim": vector_dim},
{"FieldName": "sparse_vector", "FieldType": "sparse_vector"},
@@ -53,7 +61,16 @@ class CollectionSchemas:
{"FieldName": "updated_at", "FieldType": "date_time"},
{"FieldName": "active_count", "FieldType": "int64"},
{"FieldName": "parent_uri", "FieldType": "path"},
- {"FieldName": "is_leaf", "FieldType": "bool"},
+ # level 字段:区分 L0/L1/L2 层级
+ # 枚举值:
+ # - 0 = L0(abstract,摘要)
+ # - 1 = L1(overview,概览)
+ # - 2 = L2(detail/content,详情/内容,默认)
+ # URI 命名规则:
+ # - level=0: {目录}/.abstract.md
+ # - level=1: {目录}/.overview.md
+ # - level=2: {文件路径}
+ {"FieldName": "level", "FieldType": "int64"},
{"FieldName": "name", "FieldType": "string"},
{"FieldName": "description", "FieldType": "string"},
{"FieldName": "tags", "FieldType": "string"},
@@ -67,7 +84,7 @@ class CollectionSchemas:
"updated_at",
"active_count",
"parent_uri",
- "is_leaf",
+ "level",
"name",
"tags",
],
diff --git a/openviking/storage/queuefs/embedding_msg_converter.py b/openviking/storage/queuefs/embedding_msg_converter.py
index f0980bf35..d105d5f60 100644
--- a/openviking/storage/queuefs/embedding_msg_converter.py
+++ b/openviking/storage/queuefs/embedding_msg_converter.py
@@ -7,7 +7,7 @@ This module provides a unified interface for converting Context objects
to EmbeddingMsg objects for asynchronous vector processing.
"""
-from openviking.core.context import Context
+from openviking.core.context import Context, ContextLevel
from openviking.storage.queuefs.embedding_msg import EmbeddingMsg
from openviking_cli.utils import get_logger
@@ -26,9 +26,20 @@ class EmbeddingMsgConverter:
if not vectorization_text:
return None
+ context_dict = context.to_dict()
+
+ # 根据 URI 判断 level 字段(用于向量索引)
+ uri = context_dict.get("uri", "")
+ if uri.endswith("/.abstract.md"):
+ context_dict["level"] = ContextLevel.ABSTRACT
+ elif uri.endswith("/.overview.md"):
+ context_dict["level"] = ContextLevel.OVERVIEW
+ else:
+ context_dict["level"] = ContextLevel.DETAIL
+
embedding_msg = EmbeddingMsg(
message=vectorization_text,
- context_data=context.to_dict(),
+ context_data=context_dict,
)
# Set any additional fields from kwargs
diff --git a/openviking/storage/queuefs/semantic_processor.py b/openviking/storage/queuefs/semantic_processor.py
index cfad652d2..d312c57ad 100644
--- a/openviking/storage/queuefs/semantic_processor.py
+++ b/openviking/storage/queuefs/semantic_processor.py
@@ -441,21 +441,36 @@ class SemanticProcessor(DequeueHandlerBase):
from openviking.storage.queuefs import get_queue_manager
from openviking.storage.queuefs.embedding_msg_converter import EmbeddingMsgConverter
- parent_uri = VikingURI(uri).parent.uri
- context = Context(
- uri=uri,
- parent_uri=parent_uri,
+ queue_manager = get_queue_manager()
+ embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)
+
+ # Vectorize L0: .abstract.md (abstract), level=0
+ abstract_uri = f"{uri}/.abstract.md"
+ context_abstract = Context(
+ uri=abstract_uri,
+ parent_uri=uri,
is_leaf=False,
abstract=abstract,
context_type=context_type,
)
- context.set_vectorize(Vectorize(text=overview))
-
- embedding_msg = EmbeddingMsgConverter.from_context(context)
- queue_manager = get_queue_manager()
- embedding_queue = queue_manager.get_queue(queue_manager.EMBEDDING)
- await embedding_queue.enqueue(embedding_msg) # type: ignore
- logger.debug(f"Enqueued directory for vectorization: {uri}")
+ context_abstract.set_vectorize(Vectorize(text=abstract))
+ embedding_msg_abstract = EmbeddingMsgConverter.from_context(context_abstract)
+ await embedding_queue.enqueue(embedding_msg_abstract) # type: ignore
+ logger.debug(f"Enqueued directory L0 (abstract) for vectorization: {abstract_uri}")
+
+ # Vectorize L1: .overview.md (overview), level=1
+ overview_uri = f"{uri}/.overview.md"
+ context_overview = Context(
+ uri=overview_uri,
+ parent_uri=uri,
+ is_leaf=False,
+ abstract=abstract,
+ context_type=context_type,
+ )
+ context_overview.set_vectorize(Vectorize(text=overview))
+ embedding_msg_overview = EmbeddingMsgConverter.from_context(context_overview)
+ await embedding_queue.enqueue(embedding_msg_overview) # type: ignore
+ logger.debug(f"Enqueued directory L1 (overview) for vectorization: {overview_uri}")
async def _vectorize_files(
self,
diff --git a/openviking/storage/viking_vector_index_backend.py b/openviking/storage/viking_vector_index_backend.py
index bcfbdf1c6..d75a55237 100644
--- a/openviking/storage/viking_vector_index_backend.py
+++ b/openviking/storage/viking_vector_index_backend.py
@@ -574,7 +574,7 @@ class VikingVectorIndexBackend(VikingDBInterface):
total_deleted = 0
# If any record indicates this URI is a directory node, remove descendants first.
- if any(not r.get("is_leaf", False) for r in target_records):
+ if any(r.get("level") in [0, 1] for r in target_records):
descendant_count = await self._remove_descendants(collection, uri)
total_deleted += descendant_count
@@ -602,10 +602,10 @@ class VikingVectorIndexBackend(VikingDBInterface):
for child in children:
child_uri = child.get("uri")
- is_leaf = child.get("is_leaf", False)
+ level = child.get("level", 2)
# Recursively delete if child is also an intermediate directory
- if not is_leaf and child_uri:
+ if level in [0, 1] and child_uri:
descendant_count = await self._remove_descendants(collection, child_uri)
total_deleted += descendant_count
diff --git a/openviking_cli/retrieve/types.py b/openviking_cli/retrieve/types.py
index 9d22a8d9a..db4347f0a 100644
--- a/openviking_cli/retrieve/types.py
+++ b/openviking_cli/retrieve/types.py
@@ -283,7 +283,7 @@ class MatchedContext:
uri: str
context_type: ContextType
- is_leaf: bool = False
+ level: int = 2
abstract: str = ""
overview: Optional[str] = None
category: str = ""
@@ -367,7 +367,7 @@ class FindResult:
return {
"context_type": ctx.context_type.value,
"uri": ctx.uri,
- "is_leaf": ctx.is_leaf,
+ "level": ctx.level,
"score": ctx.score,
"category": ctx.category,
"match_reason": ctx.match_reason,
@@ -393,7 +393,7 @@ class FindResult:
return MatchedContext(
uri=d.get("uri", ""),
context_type=ContextType(d.get("context_type", "resource")),
- is_leaf=d.get("is_leaf", False),
+ level=d.get("level", 2),
abstract=d.get("abstract", ""),
overview=d.get("overview"),
category=d.get("category", ""),